Передайте тестовую выборку одному локальному оценщику и сохраните отчёт

/// Owns one test epoch and consumes its local scoring permission on first use.
///
/// This type is deliberately not cloneable. It enforces one access through one
/// owner; external dataset governance is still required to prevent another
/// process from constructing a separate owner over copied data. Construction
/// checks the epoch's stored `Test` enum value, not its external lineage.
#[derive(Debug)]
pub struct FinalEvaluator {
    test_epoch: MiniBatchEpoch,
    provenance: EvaluationProvenance,
    access_count: u8,
}

impl FinalEvaluator {
    pub fn new(
        test_epoch: MiniBatchEpoch,
        provenance: EvaluationProvenance,
    ) -> Result<Self, EvaluationError> {
        if test_epoch.partition() != Partition::Test {
            return Err(EvaluationError::WrongTestPartition {
                actual: test_epoch.partition(),
            });
        }
        if test_epoch.window_count() == 0 {
            return Err(EvaluationError::EmptyTestEpoch);
        }
        if test_epoch.context_length() != provenance.context_length() {
            return Err(EvaluationError::EpochContextMismatch {
                expected: provenance.context_length(),
                actual: test_epoch.context_length(),
            });
        }
        Ok(Self {
            test_epoch,
            provenance,
            access_count: 0,
        })
    }

    pub const fn access_count(&self) -> u8 {
        self.access_count
    }

    pub fn evaluate_once(
        &mut self,
        decoder: SelectedDecoder<'_>,
        bigram: FrozenBigram<'_>,
    ) -> Result<FinalEvaluationReport, EvaluationError> {
        if self.access_count != 0 {
            return Err(EvaluationError::AlreadyEvaluated);
        }
        require_matching_provenance_assertions(&self.provenance, decoder.provenance())?;
        require_matching_provenance_assertions(&self.provenance, bigram.provenance())?;

        if !selected_state_matches_model(decoder.state(), decoder.model()) {
            return Err(EvaluationError::SelectedStateMismatch);
        }

        let model_config = decoder.model().config();
        if model_config.max_positions() != self.provenance.context_length() {
            return Err(EvaluationError::ModelContextMismatch {
                expected: self.provenance.context_length(),
                actual: model_config.max_positions(),
            });
        }
        let decoder_vocabulary = model_config.vocabulary_size();
        let bigram_vocabulary = bigram.model().vocabulary_size();
        if decoder_vocabulary != bigram_vocabulary {
            return Err(EvaluationError::VocabularyMismatch {
                decoder: decoder_vocabulary,
                bigram: bigram_vocabulary,
            });
        }

        // Every metadata error above leaves the test unopened. From this line on,
        // even an error burns the local gate because token evidence is inspected.
        self.access_count = 1;
        let inspected = InspectedTestEpoch::inspect(&self.test_epoch, decoder_vocabulary)?;
        let model = decoder.model();
        let parameters_before = parameter_bits(model);
        let gradients_before = gradient_bits(model)?;

        let measured = evaluate_no_grad(model, inspected.epoch())?;
        if measured.recorded_graphs() != 0 {
            return Err(EvaluationError::GraphRecorded {
                count: measured.recorded_graphs(),
            });
        }
        let parameters_after = parameter_bits(model);
        if parameters_after != parameters_before {
            return Err(EvaluationError::DecoderParameterChanged);
        }
        let gradients_after = gradient_bits(model)?;
        if gradients_after != gradients_before {
            return Err(EvaluationError::DecoderGradientChanged);
        }

        let decoder_score = ModelScore::new(
            EvaluatedModel::SelectedDecoder,
            measured.mean_loss() * measured.token_count() as f64,
            measured.token_count(),
            measured.mean_loss(),
            measured.mean_loss().exp(),
        )?;
        let bigram_score = score_bigram(bigram.model(), &inspected)?;
        if inspected.evidence().window_target_slot_count != measured.token_count()
            || inspected.evidence().window_target_slot_count != bigram_score.target_count()
        {
            return Err(EvaluationError::TargetCountMismatch {
                expected: inspected.evidence().window_target_slot_count,
                decoder: measured.token_count(),
                bigram: bigram_score.target_count(),
            });
        }
        let evidence = inspected.into_evidence();

        Ok(FinalEvaluationReport {
            version: FINAL_EVALUATION_REPORT_VERSION,
            selected_step: decoder.selected_step(),
            selected_validation_loss: decoder.selected_validation_loss(),
            provenance: self.provenance.clone(),
            test_document_ids: evidence.document_ids,
            window_slot_fingerprint: evidence.window_slot_fingerprint,
            window_count: self.test_epoch.window_count(),
            batch_count: self.test_epoch.batch_count(),
            window_target_slot_count: evidence.window_target_slot_count,
            document_transition_occurrence_count: evidence.document_transition_occurrence_count,
            transition_multiplicity_counts: evidence.transition_multiplicity_counts,
            decoder: decoder_score,
            bigram: bigram_score,
            access_count: self.access_count,
            recorded_graphs: measured.recorded_graphs(),
            parameters_unchanged: true,
            gradients_unchanged: true,
        })
    }
}

/// Caller-supplied identifiers and context metadata shared by report participants.
///
/// Construction proves only that the three identifier strings are nonblank and
/// the context length is positive. Equality between values of this type proves
/// only that callers supplied matching assertions; it does not hash or inspect a
/// corpus, split construction, or tokenizer. The evaluator separately checks the
/// context value against the test epoch and decoder capacity.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct EvaluationProvenance {
    corpus_fingerprint: String,
    split_fingerprint: String,
    tokenizer_fingerprint: String,
    context_length: usize,
}

impl EvaluationProvenance {
    pub fn new(
        corpus_fingerprint: impl Into<String>,
        split_fingerprint: impl Into<String>,
        tokenizer_fingerprint: impl Into<String>,
        context_length: usize,
    ) -> Result<Self, EvaluationError> {
        let provenance = Self {
            corpus_fingerprint: corpus_fingerprint.into(),
            split_fingerprint: split_fingerprint.into(),
            tokenizer_fingerprint: tokenizer_fingerprint.into(),
            context_length,
        };
        for (field, value) in [
            (ProvenanceField::Corpus, provenance.corpus_fingerprint()),
            (ProvenanceField::Split, provenance.split_fingerprint()),
            (
                ProvenanceField::Tokenizer,
                provenance.tokenizer_fingerprint(),
            ),
        ] {
            if value.trim().is_empty() {
                return Err(EvaluationError::EmptyProvenance { field });
            }
        }
        if context_length == 0 {
            return Err(EvaluationError::ZeroContextLength);
        }
        Ok(provenance)
    }

    pub fn corpus_fingerprint(&self) -> &str {
        &self.corpus_fingerprint
    }

    pub fn split_fingerprint(&self) -> &str {
        &self.split_fingerprint
    }

    pub fn tokenizer_fingerprint(&self) -> &str {
        &self.tokenizer_fingerprint
    }

    pub const fn context_length(&self) -> usize {
        self.context_length
    }
}

/// A borrowed decoder accompanied by the caller's validation-role assertion.
///
/// Construction checks the asserted role and the numeric shape of the supplied
/// selection values. It cannot reconstruct how `selected_step`,
/// `selected_validation_loss`, or the model were selected. Before test access,
/// the evaluator does mechanically compare the retained state with the borrowed
/// model's exact configuration, names, shapes, and value bits.
#[derive(Clone, Copy, Debug)]
pub struct SelectedDecoder<'a> {
    state: &'a DecoderModelState,
    model: &'a DecoderModel,
    selected_step: usize,
    selected_validation_loss: f64,
    provenance: &'a EvaluationProvenance,
}

impl<'a> SelectedDecoder<'a> {
    pub fn new(
        state: &'a DecoderModelState,
        model: &'a DecoderModel,
        selected_step: usize,
        selected_validation_loss: f64,
        selection_partition_assertion: Partition,
        provenance: &'a EvaluationProvenance,
    ) -> Result<Self, EvaluationError> {
        if selection_partition_assertion != Partition::Validation {
            return Err(EvaluationError::WrongSelectionPartition {
                actual: selection_partition_assertion,
            });
        }
        if !selected_validation_loss.is_finite() || selected_validation_loss < 0.0 {
            return Err(EvaluationError::InvalidSelectionLoss {
                value: selected_validation_loss,
            });
        }
        Ok(Self {
            state,
            model,
            selected_step,
            selected_validation_loss,
            provenance,
        })
    }

    const fn model(self) -> &'a DecoderModel {
        self.model
    }

    const fn state(self) -> &'a DecoderModelState {
        self.state
    }

    pub const fn selected_step(self) -> usize {
        self.selected_step
    }

    pub const fn selected_validation_loss(self) -> f64 {
        self.selected_validation_loss
    }

    pub const fn provenance(self) -> &'a EvaluationProvenance {
        self.provenance
    }
}

/// A borrowed count baseline accompanied by the caller's training-role assertion.
///
/// Construction checks the asserted role and that the model reports at least one
/// fitted document. It cannot discover which documents produced the counts.
#[derive(Clone, Copy, Debug)]
pub struct FrozenBigram<'a> {
    model: &'a BigramModel,
    provenance: &'a EvaluationProvenance,
}

impl<'a> FrozenBigram<'a> {
    pub fn new(
        model: &'a BigramModel,
        fit_partition_assertion: Partition,
        provenance: &'a EvaluationProvenance,
    ) -> Result<Self, EvaluationError> {
        if fit_partition_assertion != Partition::Train {
            return Err(EvaluationError::WrongBaselinePartition {
                actual: fit_partition_assertion,
            });
        }
        if model.fitted_documents() == 0 {
            return Err(EvaluationError::UnfittedBigram);
        }
        Ok(Self { model, provenance })
    }

    pub const fn model(self) -> &'a BigramModel {
        self.model
    }

    pub const fn provenance(self) -> &'a EvaluationProvenance {
        self.provenance
    }
}

#[derive(Clone, Debug, PartialEq)]
pub struct LearnerEvidence {
    pub report: FinalEvaluationReport,
    pub selection_test_partition_rejected: bool,
    pub gate_openings_before: u8,
    pub baseline_alpha: f64,
    pub baseline_documents: usize,
    pub baseline_transitions: u64,
    pub token_weighted: bool,
    pub provenance_assertions_match: bool,
    pub within_run_selection_isolated: bool,
}

/// Builds both frozen candidates before opening one owned test evaluator.
pub fn learner_evidence() -> Result<LearnerEvidence, FixtureError> {
    let selected = selection_evidence()?;
    require(
        selected.test_partition_rejected,
        "selection no longer rejects the test partition",
    )?;
    let provenance_assertions = fixture_provenance_assertions()?;
    let training_documents = fixture_training_documents();
    let baseline = BigramModel::fit_training_documents(
        VOCABULARY_SIZE,
        BIGRAM_ALPHA,
        training_documents.iter().map(|(_, tokens)| *tokens),
    )?;
    require(
        baseline.fitted_documents() == training_documents.len(),
        "baseline training document count changed",
    )?;
    require(
        baseline.fitted_transitions() == 22,
        "baseline training transition count changed",
    )?;

    let decoder = SelectedDecoder::new(
        selected.result.selected_state(),
        selected.result.selected_model(),
        selected.result.selected_step(),
        selected.result.selected_validation_loss(),
        Partition::Validation,
        &provenance_assertions,
    )?;
    let bigram = FrozenBigram::new(&baseline, Partition::Train, &provenance_assertions)?;
    let mut evaluator = FinalEvaluator::new(test_epoch()?, provenance_assertions.clone())?;
    let gate_openings_before = evaluator.access_count();
    require(gate_openings_before == 0, "test gate opened before scoring")?;
    let report = evaluator.evaluate_once(decoder, bigram)?;

    require(report.version() == 1, "report version changed")?;
    require(
        report.access_count() == 1,
        "test gate-opening count changed",
    )?;
    require(report.target_count() == 24, "test target count changed")?;
    require(report.window_count() == 12, "test window count changed")?;
    require(report.batch_count() == 3, "test batch count changed")?;
    require(
        report.test_document_ids() == ["test-a", "test-b"],
        "test document order changed",
    )?;
    require(
        report.target_fingerprint() == "fnv1a64:dac4bb4d76beeb59",
        "test evidence fingerprint changed",
    )?;
    require(
        report.decoder().target_count() == report.bigram().target_count(),
        "models did not score identical target counts",
    )?;
    require(
        report.recorded_graphs() == 0
            && report.parameters_unchanged()
            && report.gradients_unchanged(),
        "graph-free state-preservation proof changed",
    )?;
    require(
        report.decoder_has_lower_loss(),
        "fixed Chapter 34 fixture no longer records lower decoder loss than its bigram",
    )?;
    let token_weighted = [report.decoder(), report.bigram()]
        .into_iter()
        .all(|score| {
            (score.total_nll() / score.target_count() as f64 - score.mean_nll()).abs() <= 1e-12
        });
    require(token_weighted, "model scores are no longer token weighted")?;
    let provenance_assertions_match = report.provenance() == &provenance_assertions;
    require(
        provenance_assertions_match,
        "report provenance assertions no longer match the fixture assertions",
    )?;
    let within_run_selection_isolated = selected.test_partition_rejected
        && gate_openings_before == 0
        && report.access_count() == 1
        && report.recorded_graphs() == 0
        && report.parameters_unchanged()
        && report.gradients_unchanged();
    require(
        within_run_selection_isolated,
        "within-run selection isolation evidence changed",
    )?;

    Ok(LearnerEvidence {
        report,
        selection_test_partition_rejected: selected.test_partition_rejected,
        gate_openings_before,
        baseline_alpha: baseline.alpha(),
        baseline_documents: baseline.fitted_documents(),
        baseline_transitions: baseline.fitted_transitions(),
        token_weighted,
        provenance_assertions_match,
        within_run_selection_isolated,
    })
}

#[derive(Debug)]
struct TestEvidence {
    document_ids: Vec<String>,
    window_slot_fingerprint: String,
    window_target_slot_count: usize,
    document_transition_occurrence_count: usize,
    transition_multiplicity_counts: Vec<usize>,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct CheckedTokenPair {
    input: usize,
    target: usize,
}

/// One gate-opening inspection of a borrowed test epoch.
///
/// This type is private to the module. Current callers construct it only through
/// `inspect`; it exposes no mutation API and retains an immutable epoch borrow.
#[derive(Debug)]
struct InspectedTestEpoch<'a> {
    epoch: &'a MiniBatchEpoch,
    evidence: TestEvidence,
    checked_pairs: Vec<CheckedTokenPair>,
}

fn append_checked_aligned_tokens(
    batch: usize,
    inputs: &[u32],
    targets: &[u32],
    vocabulary_size: usize,
    checked_pairs: &mut Vec<CheckedTokenPair>,
) -> Result<(), EvaluationError> {
    if inputs.len() != targets.len() {
        return Err(EvaluationError::TargetAlignmentMismatch {
            batch,
            inputs: inputs.len(),
            targets: targets.len(),
        });
    }
    for (position, (&input, &target)) in inputs.iter().zip(targets).enumerate() {
        let Some(input_index) = usize::try_from(input)
            .ok()
            .filter(|id| *id < vocabulary_size)
        else {
            return Err(EvaluationError::InputTokenOutOfRange {
                batch,
                position,
                id: input,
                vocabulary_size,
            });
        };
        let Some(target_index) = usize::try_from(target)
            .ok()
            .filter(|id| *id < vocabulary_size)
        else {
            return Err(EvaluationError::TargetTokenOutOfRange {
                batch,
                position,
                id: target,
                vocabulary_size,
            });
        };
        checked_pairs.push(CheckedTokenPair {
            input: input_index,
            target: target_index,
        });
    }
    Ok(())
}

impl<'a> InspectedTestEpoch<'a> {
    fn inspect(epoch: &'a MiniBatchEpoch, vocabulary_size: usize) -> Result<Self, EvaluationError> {
        let mut document_ids = Vec::new();
        let mut window_target_slot_count = 0_usize;
        let mut transition_multiplicities = BTreeMap::<(String, usize), usize>::new();
        let mut fingerprint = 14_695_981_039_346_656_037_u64;
        let checked_pair_count = epoch
            .batches()
            .iter()
            .map(|batch| batch.targets().len())
            .sum();
        let mut checked_pairs = Vec::with_capacity(checked_pair_count);

        for (batch_index, batch) in epoch.batches().iter().enumerate() {
            append_checked_aligned_tokens(
                batch_index,
                batch.inputs(),
                batch.targets(),
                vocabulary_size,
                &mut checked_pairs,
            )?;
            for row in 0..batch.batch_width() {
                let origin = &batch.provenance()[row];
                if !document_ids
                    .iter()
                    .any(|existing| existing == origin.document_id())
                {
                    document_ids.push(origin.document_id().to_owned());
                }
                fnv1a_bytes(&mut fingerprint, origin.document_id().as_bytes());
                fnv1a_byte(&mut fingerprint, 0xff);
                fnv1a_bytes(&mut fingerprint, &(origin.start() as u64).to_le_bytes());
                let inputs = batch
                    .input_row(row)
                    .expect("batch row is constructed from complete inputs");
                let targets = batch
                    .target_row(row)
                    .expect("batch row is constructed from complete targets");
                for (slot, (&input, &target)) in inputs.iter().zip(targets).enumerate() {
                    fnv1a_bytes(&mut fingerprint, &input.to_le_bytes());
                    fnv1a_bytes(&mut fingerprint, &target.to_le_bytes());
                    window_target_slot_count += 1;
                    let absolute_target_position = origin.start() + slot + 1;
                    *transition_multiplicities
                        .entry((origin.document_id().to_owned(), absolute_target_position))
                        .or_default() += 1;
                }
            }
        }

        let document_transition_occurrence_count = transition_multiplicities.len();
        let maximum_multiplicity = transition_multiplicities
            .values()
            .copied()
            .max()
            .unwrap_or(0);
        let mut transition_multiplicity_counts = vec![0_usize; maximum_multiplicity];
        for multiplicity in transition_multiplicities.into_values() {
            transition_multiplicity_counts[multiplicity - 1] += 1;
        }
        debug_assert_eq!(window_target_slot_count, checked_pairs.len());
        debug_assert_eq!(
            window_target_slot_count,
            transition_multiplicity_counts
                .iter()
                .enumerate()
                .map(|(index, count)| (index + 1) * count)
                .sum::<usize>()
        );
        Ok(Self {
            epoch,
            evidence: TestEvidence {
                document_ids,
                window_slot_fingerprint: format!("fnv1a64:{fingerprint:016x}"),
                window_target_slot_count,
                document_transition_occurrence_count,
                transition_multiplicity_counts,
            },
            checked_pairs,
        })
    }

    const fn epoch(&self) -> &'a MiniBatchEpoch {
        self.epoch
    }

    const fn evidence(&self) -> &TestEvidence {
        &self.evidence
    }

    fn checked_pairs(&self) -> &[CheckedTokenPair] {
        &self.checked_pairs
    }

    fn into_evidence(self) -> TestEvidence {
        self.evidence
    }
}

    /// Scores indices whose vocabulary bounds were established by a crate-owned boundary.
    pub(crate) fn smoothed_probability_for_checked_indices(
        &self,
        from: usize,
        to: usize,
    ) -> Result<f64, BigramError> {
        debug_assert!(from < self.vocabulary_size);
        debug_assert!(to < self.vocabulary_size);
        let row_start = from * self.vocabulary_size;
        let numerator = self.counts[row_start + to] as f64 + self.alpha;
        let row_total = self.counts[row_start..row_start + self.vocabulary_size]
            .iter()
            .try_fold(0_u64, |total, count| {
                total
                    .checked_add(*count)
                    .ok_or(BigramError::TooManyTransitions)
            })?;
        let denominator = row_total as f64 + self.alpha * self.vocabulary_size as f64;
        Ok(numerator / denominator)
    }

fn main() {
    print!(
        "{}",
        ch34_final_evaluation::learner_report().expect("Chapter 34 fixture must remain valid")
    );
}

/// Emits the exact static evidence consumed by the Chapter 34 figure.
pub fn diagram_trace() -> Result<String, FixtureError> {
    let evidence = learner_evidence()?;
    let report = &evidence.report;
    Ok(format!(
        "FINAL_EVALUATION_TRACE_V1\n\
REPORT|version={}|partition=test|selected_step={}|selection_criterion=validation-only|gate_openings_before={}|gate_openings_after={}\n\
GATE|selection_test_partition_rejected={}\n\
PROVENANCE|corpus={}|split={}|tokenizer={}|vocabulary={}|context={}|documents={}|windows={}|batches={}|targets={}|target_fingerprint={}\n\
SCORE|model=selected-decoder|fit_partition=train|selected_by=validation|targets={}|total_nll={:.6}|mean_nll={:.6}|perplexity={:.6}\n\
SCORE|model=frozen-bigram|fit_partition=train|selected_by=none|targets={}|total_nll={:.6}|mean_nll={:.6}|perplexity={:.6}\n\
COMPARE|lower_loss=selected-decoder|loss_gap={:.6}|same_targets=true|decoder_lower_on_fixture={}|evidence_scope={}|within_run_selection_isolated={}|fixture_selected_for_ordering={}|independent_generalization_estimate={}|architecture_superiority_evidence={}\n\
PROOF|token_weighted={}|provenance_assertions_match={}|graph_nodes={}|parameters_unchanged={}|gradients_unchanged={}|selection_closed={}\n\
END_FINAL_EVALUATION_TRACE\n",
        report.version(),
        report.selected_step(),
        evidence.gate_openings_before,
        report.access_count(),
        evidence.selection_test_partition_rejected,
        report.provenance().corpus_fingerprint(),
        report.provenance().split_fingerprint(),
        report.provenance().tokenizer_fingerprint(),
        VOCABULARY_SIZE,
        report.provenance().context_length(),
        report.test_document_ids().join(","),
        report.window_count(),
        report.batch_count(),
        report.target_count(),
        report.target_fingerprint(),
        report.decoder().target_count(),
        report.decoder().total_nll(),
        report.decoder().mean_nll(),
        report.decoder().perplexity(),
        report.bigram().target_count(),
        report.bigram().total_nll(),
        report.bigram().mean_nll(),
        report.bigram().perplexity(),
        report.loss_gap(),
        report.decoder_has_lower_loss(),
        FIXED_FIXTURE_EVIDENCE_SCOPE,
        evidence.within_run_selection_isolated,
        FIXTURE_SELECTED_FOR_ORDERING,
        INDEPENDENT_GENERALIZATION_ESTIMATE,
        ARCHITECTURE_SUPERIORITY_EVIDENCE,
        evidence.token_weighted,
        evidence.provenance_assertions_match,
        report.recorded_graphs(),
        report.parameters_unchanged(),
        report.gradients_unchanged(),
        evidence.selection_test_partition_rejected,
    ))
}