diff --git a/scripts/analyze_fashionpedia_post_validation_v1.py b/scripts/analyze_fashionpedia_post_validation_v1.py index c2e76cc..7a332ec 100644 --- a/scripts/analyze_fashionpedia_post_validation_v1.py +++ b/scripts/analyze_fashionpedia_post_validation_v1.py @@ -8,17 +8,17 @@ import os import sys from collections import Counter -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np from PIL import Image, ImageDraw, ImageFont - RAW_METRICS = ( "accuracy", "balanced_accuracy", @@ -190,7 +190,9 @@ def make_contact_sheet( columns = 3 rows = max(1, math.ceil(max(1, len(examples)) / columns)) header_height = 44 - sheet = Image.new("RGB", (columns * tile_width, header_height + rows * tile_height), "white") + sheet = Image.new( + "RGB", (columns * tile_width, header_height + rows * tile_height), "white" + ) draw = ImageDraw.Draw(sheet) font = ImageFont.load_default() draw.text((10, 12), title, fill="black", font=font) @@ -212,8 +214,15 @@ def make_contact_sheet( sheet.paste(crop, (x, y)) image_status = "resolved" except Exception as exc: # noqa: BLE001 - draw.rectangle((x0 + 5, y0 + 5, x0 + tile_width - 5, y0 + 220), outline="black") - draw.text((x0 + 10, y0 + 15), f"crop unavailable: {type(exc).__name__}", fill="black", font=font) + draw.rectangle( + (x0 + 5, y0 + 5, x0 + tile_width - 5, y0 + 220), outline="black" + ) + draw.text( + (x0 + 10, y0 + 15), + f"crop unavailable: {type(exc).__name__}", + fill="black", + font=font, + ) image_status = f"unavailable:{type(exc).__name__}" lines = [ f"true: {example['true_attribute_name']}", @@ -222,7 +231,9 @@ def make_contact_sheet( f"ann: {example['annotation_id']} image: {example['image_id']}", ] for line_index, line in enumerate(lines): - draw.text((x0 + 8, y0 + 230 + 18 * line_index), line, fill="black", font=font) + draw.text( + (x0 + 8, y0 + 230 + 18 * line_index), line, fill="black", font=font + ) index_rows.append( { "contact_sheet": output.name, @@ -247,13 +258,17 @@ def main() -> None: confirmation_summary_path = Path(args.confirmation_summary).resolve() output_dir = Path(args.output_dir).resolve() if not confirmation_summary_path.is_file(): - raise FileNotFoundError(f"Missing confirmation summary: {confirmation_summary_path}") + raise FileNotFoundError( + f"Missing confirmation summary: {confirmation_summary_path}" + ) if output_dir.exists(): raise FileExistsError(f"Refusing to overwrite analysis directory: {output_dir}") summary = read_json(confirmation_summary_path) if summary.get("status") != "PASS": - raise RuntimeError(f"Final confirmation status is not PASS: {summary.get('status')!r}") + raise RuntimeError( + f"Final confirmation status is not PASS: {summary.get('status')!r}" + ) if summary.get("evaluation_scope") != "untouched_official_fashionpedia_validation": raise RuntimeError("Unexpected confirmation scope.") @@ -265,16 +280,28 @@ def main() -> None: tables_dir.mkdir() crops_dir.mkdir() - frozen_predictions_path = find_artifact(summary, "frozen_head", "confirmation_predictions_csv") - lora_predictions_path = find_artifact(summary, "lora", "confirmation_predictions_csv") - frozen_per_class_path = find_artifact(summary, "frozen_head", "raw_per_class_metrics_csv") + frozen_predictions_path = find_artifact( + summary, "frozen_head", "confirmation_predictions_csv" + ) + lora_predictions_path = find_artifact( + summary, "lora", "confirmation_predictions_csv" + ) + frozen_per_class_path = find_artifact( + summary, "frozen_head", "raw_per_class_metrics_csv" + ) lora_per_class_path = find_artifact(summary, "lora", "raw_per_class_metrics_csv") frozen_ece_path = find_artifact(summary, "frozen_head", "raw_ece_bins_json") lora_ece_path = find_artifact(summary, "lora", "raw_ece_bins_json") - validation_manifest_path = confirmation_summary_path.parent / "manifest" / "official_validation_manifest.csv" + validation_manifest_path = ( + confirmation_summary_path.parent + / "manifest" + / "official_validation_manifest.csv" + ) if not validation_manifest_path.is_file(): - raise FileNotFoundError(f"Missing official validation manifest: {validation_manifest_path}") + raise FileNotFoundError( + f"Missing official validation manifest: {validation_manifest_path}" + ) frozen_predictions = read_csv(frozen_predictions_path) lora_predictions = read_csv(lora_predictions_path) @@ -297,9 +324,15 @@ def main() -> None: target_ids = [int(value) for value in summary["data"]["class_counts"].keys()] # Preserve fixed original task order from prediction probability headers. - probability_headers = [key for key in frozen_predictions[0] if key.startswith("raw_probability_attribute_")] + probability_headers = [ + key + for key in frozen_predictions[0] + if key.startswith("raw_probability_attribute_") + ] target_ids = [int(key.rsplit("_", 1)[1]) for key in probability_headers] - target_names = {int(row["attribute_id"]): row["attribute_name"] for row in frozen_per_class} + target_names = { + int(row["attribute_id"]): row["attribute_name"] for row in frozen_per_class + } labels = [target_names[attribute_id] for attribute_id in target_ids] index_by_id = {attribute_id: index for index, attribute_id in enumerate(target_ids)} @@ -320,10 +353,13 @@ def main() -> None: frozen_matrix[index_by_id[true_id], index_by_id[frozen_id]] += 1 lora_matrix[index_by_id[true_id], index_by_id[lora_id]] += 1 transition = ( - "both_correct" if frozen_correct and lora_correct else - "lora_corrected" if (not frozen_correct and lora_correct) else - "lora_regressed" if (frozen_correct and not lora_correct) else - "both_wrong" + "both_correct" + if frozen_correct and lora_correct + else "lora_corrected" + if (not frozen_correct and lora_correct) + else "lora_regressed" + if (frozen_correct and not lora_correct) + else "both_wrong" ) transitions[transition] += 1 combined_rows.append( @@ -334,7 +370,9 @@ def main() -> None: "true_attribute_id": true_id, "true_attribute_name": frozen["true_attribute_name"], "frozen_predicted_attribute_id": frozen_id, - "frozen_predicted_attribute_name": frozen["raw_predicted_attribute_name"], + "frozen_predicted_attribute_name": frozen[ + "raw_predicted_attribute_name" + ], "frozen_confidence": float(frozen["raw_confidence"]), "frozen_correct": frozen_correct, "lora_predicted_attribute_id": lora_id, @@ -375,30 +413,82 @@ def main() -> None: "metric": metric, "frozen": float(frozen_metrics[metric]), "lora": float(lora_metrics[metric]), - "lora_minus_frozen": float(lora_metrics[metric]) - float(frozen_metrics[metric]), + "lora_minus_frozen": float(lora_metrics[metric]) + - float(frozen_metrics[metric]), } ) - write_csv(tables_dir / "official_validation_metric_comparison.csv", list(raw_metric_rows[0]), raw_metric_rows) - write_csv(tables_dir / "official_validation_per_class_comparison.csv", list(per_class_rows[0]), per_class_rows) - transition_rows = [{"transition": key, "count": value, "fraction": value / len(combined_rows)} for key, value in sorted(transitions.items())] - write_csv(tables_dir / "prediction_transition_summary.csv", ["transition", "count", "fraction"], transition_rows) - write_csv(tables_dir / "paired_official_validation_predictions.csv", list(combined_rows[0]), combined_rows) + write_csv( + tables_dir / "official_validation_metric_comparison.csv", + list(raw_metric_rows[0]), + raw_metric_rows, + ) + write_csv( + tables_dir / "official_validation_per_class_comparison.csv", + list(per_class_rows[0]), + per_class_rows, + ) + transition_rows = [ + {"transition": key, "count": value, "fraction": value / len(combined_rows)} + for key, value in sorted(transitions.items()) + ] + write_csv( + tables_dir / "prediction_transition_summary.csv", + ["transition", "count", "fraction"], + transition_rows, + ) + write_csv( + tables_dir / "paired_official_validation_predictions.csv", + list(combined_rows[0]), + combined_rows, + ) - plot_per_class_f1(per_class_rows, figures_dir / "official_validation_per_class_f1.png") - plot_confusion(frozen_matrix, labels, "Frozen SigLIP2: raw confusion matrix", figures_dir / "official_validation_frozen_confusion_matrix.png") - plot_confusion(lora_matrix, labels, "LoRA SigLIP2: raw confusion matrix", figures_dir / "official_validation_lora_confusion_matrix.png") - plot_reliability(frozen_ece, lora_ece, figures_dir / "official_validation_raw_reliability.png") + plot_per_class_f1( + per_class_rows, figures_dir / "official_validation_per_class_f1.png" + ) + plot_confusion( + frozen_matrix, + labels, + "Frozen SigLIP2: raw confusion matrix", + figures_dir / "official_validation_frozen_confusion_matrix.png", + ) + plot_confusion( + lora_matrix, + labels, + "LoRA SigLIP2: raw confusion matrix", + figures_dir / "official_validation_lora_confusion_matrix.png", + ) + plot_reliability( + frozen_ece, lora_ece, figures_dir / "official_validation_raw_reliability.png" + ) max_examples = max(1, int(args.max_examples_per_group)) examples_by_transition: dict[str, list[dict[str, Any]]] = {} for transition in ("lora_corrected", "lora_regressed", "both_wrong"): candidates = [row for row in combined_rows if row["transition"] == transition] if transition == "lora_corrected": - candidates.sort(key=lambda row: (-row["lora_confidence"], row["frozen_confidence"], int(row["annotation_id"]))) + candidates.sort( + key=lambda row: ( + -row["lora_confidence"], + row["frozen_confidence"], + int(row["annotation_id"]), + ) + ) elif transition == "lora_regressed": - candidates.sort(key=lambda row: (-row["frozen_confidence"], row["lora_confidence"], int(row["annotation_id"]))) + candidates.sort( + key=lambda row: ( + -row["frozen_confidence"], + row["lora_confidence"], + int(row["annotation_id"]), + ) + ) else: - candidates.sort(key=lambda row: (-row["lora_confidence"], -row["frozen_confidence"], int(row["annotation_id"]))) + candidates.sort( + key=lambda row: ( + -row["lora_confidence"], + -row["frozen_confidence"], + int(row["annotation_id"]), + ) + ) examples_by_transition[transition] = candidates[:max_examples] crop_index_rows: list[dict[str, Any]] = [] @@ -417,7 +507,11 @@ def main() -> None: ) ) if crop_index_rows: - write_csv(crops_dir / "representative_case_index.csv", list(crop_index_rows[0]), crop_index_rows) + write_csv( + crops_dir / "representative_case_index.csv", + list(crop_index_rows[0]), + crop_index_rows, + ) metric_lookup = {row["metric"]: row for row in raw_metric_rows} report_lines = [ @@ -439,7 +533,9 @@ def main() -> None: "", ] for row in transition_rows: - report_lines.append(f"- {row['transition']}: {row['count']} ({row['fraction']:.1%})") + report_lines.append( + f"- {row['transition']}: {row['count']} ({row['fraction']:.1%})" + ) report_lines += [ "", "## Interpretation guardrails", @@ -459,13 +555,15 @@ def main() -> None: "- `figures/official_validation_raw_reliability.png`", "- `representative_crops/`", ] - (output_dir / "POST_VALIDATION_ANALYSIS_REPORT.md").write_text("\n".join(report_lines) + "\n", encoding="utf-8") + (output_dir / "POST_VALIDATION_ANALYSIS_REPORT.md").write_text( + "\n".join(report_lines) + "\n", encoding="utf-8" + ) output_files = sorted(path for path in output_dir.rglob("*") if path.is_file()) artifact_manifest = { "status": "PASS", "schema_version": "fashionpedia_post_validation_analysis_v1", - "created_utc": datetime.now(timezone.utc).isoformat(), + "created_utc": datetime.now(UTC).isoformat(), "analysis_scope": "read_only_post_hoc_descriptive_analysis", "no_model_runtime_loaded": True, "no_inference_performed": True, @@ -483,7 +581,11 @@ def main() -> None: "official_validation_samples": len(combined_rows), "prediction_transition_counts": dict(transitions), "files": [ - {"relative_path": str(path.relative_to(output_dir)), "sha256": sha256_file(path), "bytes": path.stat().st_size} + { + "relative_path": str(path.relative_to(output_dir)), + "sha256": sha256_file(path), + "bytes": path.stat().st_size, + } for path in output_files ], } @@ -495,8 +597,14 @@ def main() -> None: print(f"LORA_CORRECTED: {transitions['lora_corrected']}", flush=True) print(f"LORA_REGRESSED: {transitions['lora_regressed']}", flush=True) print(f"BOTH_WRONG: {transitions['both_wrong']}", flush=True) - print(f"ANALYSIS_REPORT: {output_dir / 'POST_VALIDATION_ANALYSIS_REPORT.md'}", flush=True) - print(f"ANALYSIS_MANIFEST: {output_dir / 'analysis_artifact_manifest.json'}", flush=True) + print( + f"ANALYSIS_REPORT: {output_dir / 'POST_VALIDATION_ANALYSIS_REPORT.md'}", + flush=True, + ) + print( + f"ANALYSIS_MANIFEST: {output_dir / 'analysis_artifact_manifest.json'}", + flush=True, + ) if __name__ == "__main__": diff --git a/scripts/build_fashionpedia_neckline_train_dev_manifest.py b/scripts/build_fashionpedia_neckline_train_dev_manifest.py index a605434..36c2804 100644 --- a/scripts/build_fashionpedia_neckline_train_dev_manifest.py +++ b/scripts/build_fashionpedia_neckline_train_dev_manifest.py @@ -6,7 +6,7 @@ import json import random from collections import Counter, defaultdict -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path @@ -57,16 +57,12 @@ def objective( target_total: float, ) -> float: class_loss = sum( - ( - (float(counts[attribute_id]) - target_count) - / max(target_count, 1.0) - ) ** 2 + ((float(counts[attribute_id]) - target_count) / max(target_count, 1.0)) ** 2 for attribute_id, target_count in target_counts.items() ) total_loss = ( - (float(sum(counts.values())) - target_total) - / max(target_total, 1.0) + (float(sum(counts.values())) - target_total) / max(target_total, 1.0) ) ** 2 overshoot_loss = sum( @@ -115,14 +111,10 @@ def main() -> None: target_category_name = str(config["target_category"]["expected_name"]) target_attribute_rows = config["target_attributes"] - target_attribute_ids = { - int(row["id"]) - for row in target_attribute_rows - } + target_attribute_ids = {int(row["id"]) for row in target_attribute_rows} target_attribute_names = { - int(row["id"]): str(row["name"]) - for row in target_attribute_rows + int(row["id"]): str(row["name"]) for row in target_attribute_rows } development_policy = config["development_split_policy"] @@ -131,13 +123,10 @@ def main() -> None: seed = int(development_policy["seed"]) if abs((train_fraction + dev_fraction) - 1.0) > 1e-9: - raise ValueError( - "Train and development fractions must sum to 1.0." - ) + raise ValueError("Train and development fractions must sum to 1.0.") category_names = { - int(row["id"]): str(row["name"]) - for row in payload.get("categories", []) + int(row["id"]): str(row["name"]) for row in payload.get("categories", []) } observed_category_name = category_names.get(target_category_id) @@ -150,8 +139,7 @@ def main() -> None: ) image_rows = { - int(row["id"]): str(row["file_name"]) - for row in payload.get("images", []) + int(row["id"]): str(row["file_name"]) for row in payload.get("images", []) } records_by_group: dict[int, list[dict]] = defaultdict(list) @@ -230,16 +218,10 @@ def main() -> None: rng = random.Random(seed) group_ids = list(records_by_group) - tie_break = { - group_id: rng.random() - for group_id in group_ids - } + tie_break = {group_id: rng.random() for group_id in group_ids} group_class_counts: dict[int, Counter[int]] = { - group_id: Counter( - record["attribute_id"] - for record in records - ) + group_id: Counter(record["attribute_id"] for record in records) for group_id, records in records_by_group.items() } @@ -257,10 +239,7 @@ def main() -> None: for attribute_id in sorted(target_attribute_ids) } - target_dev_total = ( - float(sum(class_total_counts.values())) - * dev_fraction - ) + target_dev_total = float(sum(class_total_counts.values())) * dev_fraction dev_group_ids: set[int] = set() dev_counts: Counter[int] = Counter() @@ -288,9 +267,7 @@ def main() -> None: train_group_ids = set(group_ids) - dev_group_ids if not train_group_ids or not dev_group_ids: - raise RuntimeError( - "Degenerate split produced an empty train or dev partition." - ) + raise RuntimeError("Degenerate split produced an empty train or dev partition.") train_records = [ record @@ -304,15 +281,9 @@ def main() -> None: for record in records_by_group[group_id] ] - train_counts = Counter( - int(record["attribute_id"]) - for record in train_records - ) + train_counts = Counter(int(record["attribute_id"]) for record in train_records) - dev_counts = Counter( - int(record["attribute_id"]) - for record in dev_records - ) + dev_counts = Counter(int(record["attribute_id"]) for record in dev_records) overlap = train_group_ids & dev_group_ids @@ -339,40 +310,22 @@ def main() -> None: } ) - overall_dev_fraction = ( - float(len(dev_records)) - / float(len(train_records) + len(dev_records)) + overall_dev_fraction = float(len(dev_records)) / float( + len(train_records) + len(dev_records) ) - per_class_minimum_train = min( - row["train_count"] - for row in class_rows - ) + per_class_minimum_train = min(row["train_count"] for row in class_rows) - per_class_minimum_dev = min( - row["dev_count"] - for row in class_rows - ) + per_class_minimum_dev = min(row["dev_count"] for row in class_rows) group_disjoint_pass = len(overlap) == 0 - all_classes_in_train_pass = all( - row["train_count"] > 0 - for row in class_rows - ) + all_classes_in_train_pass = all(row["train_count"] > 0 for row in class_rows) - all_classes_in_dev_pass = all( - row["dev_count"] > 0 - for row in class_rows - ) + all_classes_in_dev_pass = all(row["dev_count"] > 0 for row in class_rows) - dev_fraction_pass = ( - abs(overall_dev_fraction - dev_fraction) <= 0.035 - ) + dev_fraction_pass = abs(overall_dev_fraction - dev_fraction) <= 0.035 - class_fraction_pass = all( - 0.12 <= row["dev_fraction"] <= 0.28 - for row in class_rows - ) + class_fraction_pass = all(0.12 <= row["dev_fraction"] <= 0.28 for row in class_rows) image_resolution_pass = skipped_missing_image == 0 @@ -424,7 +377,7 @@ def main() -> None: summary = { "status": status, "schema_version": "fashionpedia_neckline_attribute_manifest_v1", - "created_utc": datetime.now(timezone.utc).isoformat(), + "created_utc": datetime.now(UTC).isoformat(), "selection_scope": "train_only", "annotation_path": str(annotation_path), "annotation_sha256": sha256_file(annotation_path), @@ -502,12 +455,12 @@ def main() -> None: for row in class_rows: print( - f'{row["attribute_id"]:>3} ' - f'{row["total_count"]:>5} ' - f'{row["train_count"]:>5} ' - f'{row["dev_count"]:>4} ' - f'{row["dev_fraction"]:>12.3f} ' - f'{row["attribute_name"]}' + f"{row['attribute_id']:>3} " + f"{row['total_count']:>5} " + f"{row['train_count']:>5} " + f"{row['dev_count']:>4} " + f"{row['dev_fraction']:>12.3f} " + f"{row['attribute_name']}" ) print("\n===== MANIFEST CHECKS =====") @@ -523,4 +476,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/fashionpedia_neckline_source_rule_forensics_v1.py b/scripts/fashionpedia_neckline_source_rule_forensics_v1.py index 8aefd7a..e0349da 100644 --- a/scripts/fashionpedia_neckline_source_rule_forensics_v1.py +++ b/scripts/fashionpedia_neckline_source_rule_forensics_v1.py @@ -6,9 +6,10 @@ import itertools import json from collections import Counter, defaultdict -from datetime import datetime, timezone +from collections.abc import Callable +from datetime import UTC, datetime from pathlib import Path -from typing import Any, Callable +from typing import Any TARGET_IDS_DEFAULT = [182, 183, 185, 187, 189, 190, 200] @@ -109,7 +110,9 @@ def annotation_row( "bbox_valid": valid_bbox(annotation), "segmentation_kind": segmentation_kind(annotation), "segmentation_nonempty": has_nonempty_segmentation(annotation), - "all_attribute_ids": [int(value) for value in raw_attributes if str(value).lstrip("-").isdigit()], + "all_attribute_ids": [ + int(value) for value in raw_attributes if str(value).lstrip("-").isdigit() + ], "target_attribute_hits": target_hits(annotation, target_ids), } @@ -213,14 +216,18 @@ def main() -> None: train_annotation_path = Path(args.source_train_annotation).resolve() output_path = Path(args.output_json).resolve() task_config_path = Path(args.task_config).resolve() if args.task_config else None - task_audit_path = Path(args.task_audit_json).resolve() if args.task_audit_json else None + task_audit_path = ( + Path(args.task_audit_json).resolve() if args.task_audit_json else None + ) for path in (train_manifest_path, dev_manifest_path, train_annotation_path): if not path.is_file(): raise FileNotFoundError(f"Required source artifact missing: {path}") for optional_path in (task_config_path, task_audit_path): if optional_path is not None and not optional_path.is_file(): - raise FileNotFoundError(f"Optional source artifact was requested but missing: {optional_path}") + raise FileNotFoundError( + f"Optional source artifact was requested but missing: {optional_path}" + ) if output_path.exists(): raise FileExistsError(f"Refusing to overwrite forensic output: {output_path}") output_path.parent.mkdir(parents=True, exist_ok=True) @@ -234,15 +241,18 @@ def main() -> None: ): missing = sorted(required_columns - set(rows[0])) if missing: - raise RuntimeError(f"{name} source manifest missing required columns: {missing}") + raise RuntimeError( + f"{name} source manifest missing required columns: {missing}" + ) source_rows = train_rows + dev_rows source_pairs = { - canonical_pair(row["annotation_id"], row["attribute_id"]) - for row in source_rows + canonical_pair(row["annotation_id"], row["attribute_id"]) for row in source_rows } if len(source_pairs) != len(source_rows): - raise RuntimeError("Source manifests contain duplicate annotation/attribute pairs.") + raise RuntimeError( + "Source manifests contain duplicate annotation/attribute pairs." + ) target_ids = sorted({int(row["attribute_id"]) for row in source_rows}) if target_ids != TARGET_IDS_DEFAULT: @@ -255,7 +265,9 @@ def main() -> None: annotations = payload.get("annotations", []) categories = payload.get("categories", []) if not isinstance(annotations, list) or not isinstance(categories, list): - raise RuntimeError("Fashionpedia train annotation JSON has invalid annotations/categories fields.") + raise RuntimeError( + "Fashionpedia train annotation JSON has invalid annotations/categories fields." + ) category_names = {int(row["id"]): str(row.get("name", "")) for row in categories} annotation_by_id = {int(row["id"]): row for row in annotations} if len(annotation_by_id) != len(annotations): @@ -272,7 +284,9 @@ def main() -> None: source_annotation_ids = {pair[0] for pair in source_pairs} baseline_annotation_ids = {pair[0] for pair in baseline_pairs} if source_annotation_ids - set(annotation_by_id): - raise RuntimeError("Source manifests reference annotation IDs absent from source train JSON.") + raise RuntimeError( + "Source manifests reference annotation IDs absent from source train JSON." + ) source_records = [ annotation_row(annotation_by_id[annotation_id], target_set, category_names) @@ -312,7 +326,9 @@ def main() -> None: annotations, target_set, source_pairs, - lambda annotation, combo=combo: all(rule(annotation) for _, rule in combo), + lambda annotation, combo=combo: all( + rule(annotation) for _, rule in combo + ), data_derived=False, ) ) @@ -323,7 +339,9 @@ def main() -> None: annotations, target_set, source_pairs, - lambda annotation: int(annotation.get("category_id", -1)) in source_category_ids, + lambda annotation: ( + int(annotation.get("category_id", -1)) in source_category_ids + ), data_derived=True, ) ) @@ -340,7 +358,10 @@ def main() -> None: ] config_hints: dict[str, Any] = {} - for label, optional_path in (("task_config", task_config_path), ("task_audit_json", task_audit_path)): + for label, optional_path in ( + ("task_config", task_config_path), + ("task_audit_json", task_audit_path), + ): if optional_path is None: continue loaded = load_json(optional_path) @@ -353,7 +374,7 @@ def main() -> None: payload_out = { "status": "PASS", "schema_version": "fashionpedia_neckline_source_rule_forensics_v1", - "created_utc": datetime.now(timezone.utc).isoformat(), + "created_utc": datetime.now(UTC).isoformat(), "scope": "source_train_only_forensics_no_validation_access_no_model_loading", "validation_annotation_loaded": False, "validation_images_scanned": False, @@ -375,8 +396,12 @@ def main() -> None: "source_pair_reproduction_pass": source_pairs == baseline_pairs, "missing_source_pair_count": len(missing_source_pairs), "unexpected_pair_count": len(unexpected_pairs), - "missing_source_examples": [list(pair) for pair in sorted(missing_source_pairs)[:20]], - "unexpected_pair_examples": [list(pair) for pair in sorted(unexpected_pairs)[:20]], + "missing_source_examples": [ + list(pair) for pair in sorted(missing_source_pairs)[:20] + ], + "unexpected_pair_examples": [ + list(pair) for pair in sorted(unexpected_pairs)[:20] + ], }, "unexpected_pair_forensics": { "record_count": len(unexpected_records), @@ -384,8 +409,12 @@ def main() -> None: "category_id_counts": count_by(unexpected_records, "category_id"), "category_name_counts": count_by(unexpected_records, "category_name"), "iscrowd_counts": count_by(unexpected_records, "iscrowd"), - "segmentation_kind_counts": count_by(unexpected_records, "segmentation_kind"), - "segmentation_nonempty_counts": count_by(unexpected_records, "segmentation_nonempty"), + "segmentation_kind_counts": count_by( + unexpected_records, "segmentation_kind" + ), + "segmentation_nonempty_counts": count_by( + unexpected_records, "segmentation_nonempty" + ), }, "source_pair_forensics": { "record_count": len(source_records), @@ -393,7 +422,9 @@ def main() -> None: "category_name_counts": count_by(source_records, "category_name"), "iscrowd_counts": count_by(source_records, "iscrowd"), "segmentation_kind_counts": count_by(source_records, "segmentation_kind"), - "segmentation_nonempty_counts": count_by(source_records, "segmentation_nonempty"), + "segmentation_nonempty_counts": count_by( + source_records, "segmentation_nonempty" + ), "observed_category_ids": sorted(source_category_ids), }, "candidate_filter_tests": filter_reports, @@ -416,8 +447,12 @@ def main() -> None: print(f"BASELINE_RECONSTRUCTED_PAIRS: {len(baseline_pairs)}", flush=True) print(f"MISSING_SOURCE_PAIRS: {len(missing_source_pairs)}", flush=True) print(f"UNEXPECTED_RECONSTRUCTED_PAIRS: {len(unexpected_pairs)}", flush=True) - print(f"EXACT_NON_DATA_DERIVED_FILTER_MATCHES: {exact_non_data_matches}", flush=True) - print(f"EXACT_DATA_DERIVED_FILTER_MATCHES: {exact_data_derived_matches}", flush=True) + print( + f"EXACT_NON_DATA_DERIVED_FILTER_MATCHES: {exact_non_data_matches}", flush=True + ) + print( + f"EXACT_DATA_DERIVED_FILTER_MATCHES: {exact_data_derived_matches}", flush=True + ) print("VALIDATION_ANNOTATION_LOADED: False", flush=True) print("VALIDATION_IMAGES_SCANNED: False", flush=True) print("MODEL_LOADED: False", flush=True) diff --git a/scripts/final_confirmation_fashionpedia_neckline_v1.py b/scripts/final_confirmation_fashionpedia_neckline_v1.py index 644e343..6567132 100644 --- a/scripts/final_confirmation_fashionpedia_neckline_v1.py +++ b/scripts/final_confirmation_fashionpedia_neckline_v1.py @@ -10,17 +10,19 @@ import sys import time from collections import Counter -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any import numpy as np +import peft import scipy -from scipy.optimize import minimize_scalar import sklearn import torch -from PIL import Image +import transformers from peft import PeftModel +from PIL import Image +from scipy.optimize import minimize_scalar from sklearn.metrics import ( accuracy_score, balanced_accuracy_score, @@ -30,9 +32,6 @@ precision_recall_fscore_support, ) from transformers import AutoImageProcessor, AutoModel -import peft -import transformers - EPS = 1e-12 TARGET_IDS_DEFAULT = [182, 183, 185, 187, 189, 190, 200] @@ -69,7 +68,9 @@ def write_csv(path: Path, fieldnames: list[str], rows: list[dict[str, Any]]) -> writer.writerows(rows) -def required_columns(rows: list[dict[str, str]], required: set[str], artifact: Path) -> None: +def required_columns( + rows: list[dict[str, str]], required: set[str], artifact: Path +) -> None: missing = sorted(required - set(rows[0])) if missing: raise RuntimeError(f"{artifact} is missing required columns: {missing}") @@ -246,7 +247,9 @@ def crop_from_record(record: dict[str, Any], padding_fraction: float) -> Image.I width = float(record["bbox_width"]) height = float(record["bbox_height"]) if width <= 0.0 or height <= 0.0: - raise ValueError(f"Non-positive bbox for annotation_id={record['annotation_id']}") + raise ValueError( + f"Non-positive bbox for annotation_id={record['annotation_id']}" + ) with Image.open(image_path) as source: image = source.convert("RGB") image_width, image_height = image.size @@ -257,7 +260,9 @@ def crop_from_record(record: dict[str, Any], padding_fraction: float) -> Image.I right = min(image_width, int(math.ceil(x + width + pad_x))) bottom = min(image_height, int(math.ceil(y + height + pad_y))) if right <= left or bottom <= top: - raise ValueError(f"Degenerate crop for annotation_id={record['annotation_id']}") + raise ValueError( + f"Degenerate crop for annotation_id={record['annotation_id']}" + ) return image.crop((left, top, right, bottom)) @@ -285,7 +290,9 @@ def softmax(logits: np.ndarray) -> np.ndarray: return probabilities -def top_label_ece(probabilities: np.ndarray, targets: np.ndarray, bins: int) -> tuple[float, list[dict[str, Any]]]: +def top_label_ece( + probabilities: np.ndarray, targets: np.ndarray, bins: int +) -> tuple[float, list[dict[str, Any]]]: predictions = probabilities.argmax(axis=1) confidence = probabilities.max(axis=1) correct = (predictions == targets).astype(np.float64) @@ -331,7 +338,9 @@ def metric_bundle( accuracy = float(accuracy_score(labels, predictions)) balanced_accuracy = float(balanced_accuracy_score(labels, predictions)) macro_f1 = float(f1_score(labels, predictions, average="macro", zero_division=0)) - weighted_f1 = float(f1_score(labels, predictions, average="weighted", zero_division=0)) + weighted_f1 = float( + f1_score(labels, predictions, average="weighted", zero_division=0) + ) nll = float(log_loss(labels, probabilities, labels=list(range(n_classes)))) one_hot = np.eye(n_classes, dtype=np.float64)[labels] brier = float(np.mean(np.sum((probabilities - one_hot) ** 2, axis=1))) @@ -377,12 +386,21 @@ def probabilities_from_saved_dev_predictions( target_ids: list[int], ) -> tuple[np.ndarray, np.ndarray]: rows = load_csv(path) - required = {"true_attribute_id"} | {f"probability_attribute_{attr_id}" for attr_id in target_ids} + required = {"true_attribute_id"} | { + f"probability_attribute_{attr_id}" for attr_id in target_ids + } required_columns(rows, required, path) - target_to_index = {attribute_id: index for index, attribute_id in enumerate(target_ids)} - labels = np.asarray([target_to_index[int(row["true_attribute_id"])] for row in rows], dtype=np.int64) + target_to_index = { + attribute_id: index for index, attribute_id in enumerate(target_ids) + } + labels = np.asarray( + [target_to_index[int(row["true_attribute_id"])] for row in rows], dtype=np.int64 + ) probabilities = np.asarray( - [[float(row[f"probability_attribute_{attr_id}"]) for attr_id in target_ids] for row in rows], + [ + [float(row[f"probability_attribute_{attr_id}"]) for attr_id in target_ids] + for row in rows + ], dtype=np.float64, ) if not np.isfinite(probabilities).all() or np.any(probabilities < 0.0): @@ -396,13 +414,19 @@ def probabilities_from_saved_dev_predictions( def fit_temperature(probabilities: np.ndarray, labels: np.ndarray) -> dict[str, Any]: base_logits = np.log(np.clip(probabilities, EPS, 1.0)) - before = float(log_loss(labels, softmax(base_logits), labels=list(range(probabilities.shape[1])))) + before = float( + log_loss( + labels, softmax(base_logits), labels=list(range(probabilities.shape[1])) + ) + ) def objective(temperature: float) -> float: if not np.isfinite(temperature) or temperature <= 0.0: return float("inf") calibrated = softmax(base_logits / float(temperature)) - return float(log_loss(labels, calibrated, labels=list(range(probabilities.shape[1])))) + return float( + log_loss(labels, calibrated, labels=list(range(probabilities.shape[1]))) + ) result = minimize_scalar( objective, @@ -415,7 +439,9 @@ def objective(temperature: float) -> float: temperature = float(result.x) after = objective(temperature) calibrated = softmax(base_logits / temperature) - predictions_unchanged = bool(np.array_equal(probabilities.argmax(axis=1), calibrated.argmax(axis=1))) + predictions_unchanged = bool( + np.array_equal(probabilities.argmax(axis=1), calibrated.argmax(axis=1)) + ) return { "method": "single_scalar_temperature_scaling", "optimization_scope": "selected_development_predictions_only", @@ -449,7 +475,9 @@ def predict_logits( chunks: list[np.ndarray] = [] for start in range(0, len(records), batch_size): end = min(start + batch_size, len(records)) - inputs = make_inputs(records[start:end], image_processor, device, padding_fraction) + inputs = make_inputs( + records[start:end], image_processor, device, padding_fraction + ) with torch.inference_mode(): with torch.autocast(device_type="cuda", dtype=torch.bfloat16): features = encoder.get_image_features(**inputs) @@ -465,7 +493,9 @@ def predict_logits( return logits -def load_head(checkpoint_path: Path, input_dimension: int, class_count: int, device: torch.device) -> torch.nn.Module: +def load_head( + checkpoint_path: Path, input_dimension: int, class_count: int, device: torch.device +) -> torch.nn.Module: checkpoint = torch.load(checkpoint_path, map_location="cpu") if "head_state_dict" not in checkpoint: raise RuntimeError(f"Missing head_state_dict in {checkpoint_path}") @@ -499,11 +529,15 @@ def load_frozen_arm( for parameter in model.parameters(): parameter.requires_grad = False head = load_head(checkpoint_path, input_dimension, class_count, device) - return model, head, { - "model_class": model.__class__.__name__, - "adapter_parameter_count": 0, - "checkpoint_path": str(checkpoint_path), - } + return ( + model, + head, + { + "model_class": model.__class__.__name__, + "adapter_parameter_count": 0, + "checkpoint_path": str(checkpoint_path), + }, + ) def load_lora_arm( @@ -528,13 +562,17 @@ def load_lora_arm( parameter.requires_grad = False encoder = lora_model.get_base_model() head = load_head(checkpoint_path, input_dimension, class_count, device) - return encoder, head, { - "model_class": encoder.__class__.__name__, - "adapter_model_class": lora_model.__class__.__name__, - "adapter_parameter_count": count_lora_parameters(lora_model), - "checkpoint_path": str(checkpoint_path), - "adapter_directory": str(adapter_dir), - } + return ( + encoder, + head, + { + "model_class": encoder.__class__.__name__, + "adapter_model_class": lora_model.__class__.__name__, + "adapter_parameter_count": count_lora_parameters(lora_model), + "checkpoint_path": str(checkpoint_path), + "adapter_directory": str(adapter_dir), + }, + ) def prediction_rows( @@ -563,13 +601,19 @@ def prediction_rows( "raw_confidence": float(raw_probabilities[index].max()), "raw_correct": bool(raw_indices[index] == labels[index]), "calibrated_predicted_attribute_id": calibrated_attribute_id, - "calibrated_predicted_attribute_name": target_names[calibrated_attribute_id], + "calibrated_predicted_attribute_name": target_names[ + calibrated_attribute_id + ], "calibrated_confidence": float(calibrated_probabilities[index].max()), "calibrated_correct": bool(calibrated_indices[index] == labels[index]), } for class_index, attribute_id in enumerate(target_ids): - row[f"raw_probability_attribute_{attribute_id}"] = float(raw_probabilities[index, class_index]) - row[f"calibrated_probability_attribute_{attribute_id}"] = float(calibrated_probabilities[index, class_index]) + row[f"raw_probability_attribute_{attribute_id}"] = float( + raw_probabilities[index, class_index] + ) + row[f"calibrated_probability_attribute_{attribute_id}"] = float( + calibrated_probabilities[index, class_index] + ) rows.append(row) return rows @@ -597,9 +641,13 @@ def confirmation_arm( head_path = Path(str(artifacts.get("head_state_path", ""))) dev_predictions_path = Path(str(artifacts.get("best_dev_predictions_csv", ""))) if not head_path.is_file(): - raise FileNotFoundError(f"Selected head checkpoint is missing for {arm_name}: {head_path}") + raise FileNotFoundError( + f"Selected head checkpoint is missing for {arm_name}: {head_path}" + ) if not dev_predictions_path.is_file(): - raise FileNotFoundError(f"Selected development predictions are missing for {arm_name}: {dev_predictions_path}") + raise FileNotFoundError( + f"Selected development predictions are missing for {arm_name}: {dev_predictions_path}" + ) input_dimension = 768 class_count = len(target_ids) @@ -612,12 +660,20 @@ def confirmation_arm( if not adapter_dir.is_dir(): raise FileNotFoundError(f"Selected LoRA adapter is missing: {adapter_dir}") encoder, head, load_info = load_lora_arm( - model_id, cache_dir, adapter_dir, head_path, input_dimension, class_count, device + model_id, + cache_dir, + adapter_dir, + head_path, + input_dimension, + class_count, + device, ) else: raise ValueError(f"Unknown arm: {arm_name}") - dev_probabilities, dev_labels = probabilities_from_saved_dev_predictions(dev_predictions_path, target_ids) + dev_probabilities, dev_labels = probabilities_from_saved_dev_predictions( + dev_predictions_path, target_ids + ) temperature_info = fit_temperature(dev_probabilities, dev_labels) temperature = float(temperature_info["temperature"]) @@ -636,12 +692,21 @@ def confirmation_arm( raw_prediction_indices = raw_probabilities.argmax(axis=1) calibrated_prediction_indices = calibrated_probabilities.argmax(axis=1) - calibration_argmax_unchanged = bool(np.array_equal(raw_prediction_indices, calibrated_prediction_indices)) + calibration_argmax_unchanged = bool( + np.array_equal(raw_prediction_indices, calibrated_prediction_indices) + ) if not calibration_argmax_unchanged: - raise RuntimeError("Temperature scaling unexpectedly changed final argmax predictions.") + raise RuntimeError( + "Temperature scaling unexpectedly changed final argmax predictions." + ) rows = prediction_rows( - records, labels, raw_probabilities, calibrated_probabilities, target_ids, target_names + records, + labels, + raw_probabilities, + calibrated_probabilities, + target_ids, + target_names, ) prediction_fields = list(rows[0].keys()) predictions_path = arm_dir / "confirmation_predictions.csv" @@ -658,8 +723,22 @@ def confirmation_arm( ["attribute_id", "attribute_name", "precision", "recall", "f1", "support"], per_class, ) - write_json(raw_ece_path, {"top_label_ece": raw_metrics["top_label_ece"], "bin_count": ece_bins, "bins": raw_ece_rows}) - write_json(calibrated_ece_path, {"top_label_ece": calibrated_metrics["top_label_ece"], "bin_count": ece_bins, "bins": calibrated_ece_rows}) + write_json( + raw_ece_path, + { + "top_label_ece": raw_metrics["top_label_ece"], + "bin_count": ece_bins, + "bins": raw_ece_rows, + }, + ) + write_json( + calibrated_ece_path, + { + "top_label_ece": calibrated_metrics["top_label_ece"], + "bin_count": ece_bins, + "bins": calibrated_ece_rows, + }, + ) write_json(temperature_path, temperature_info) np.save(logits_path, raw_logits.astype(np.float32, copy=False)) @@ -670,16 +749,19 @@ def confirmation_arm( "raw_prediction_count_matches_confirmation": len(rows) == len(records), "temperature_optimizer_success": bool(temperature_info["optimizer_success"]), "temperature_positive": temperature > 0.0, - "temperature_preserves_argmax_on_development": bool(temperature_info["development_argmax_unchanged"]), + "temperature_preserves_argmax_on_development": bool( + temperature_info["development_argmax_unchanged"] + ), "temperature_preserves_argmax_on_confirmation": calibration_argmax_unchanged, "finite_raw_metrics": all(np.isfinite(value) for value in raw_metrics.values()), - "finite_calibrated_metrics": all(np.isfinite(value) for value in calibrated_metrics.values()), + "finite_calibrated_metrics": all( + np.isfinite(value) for value in calibrated_metrics.values() + ), } if arm_name == "lora": - qualification_checks["adapter_parameter_count_matches_development"] = ( - int(load_info["adapter_parameter_count"]) - == int(development_arm_summary["parameter_counts"]["lora_adapter"]) - ) + qualification_checks["adapter_parameter_count_matches_development"] = int( + load_info["adapter_parameter_count"] + ) == int(development_arm_summary["parameter_counts"]["lora_adapter"]) status = "PASS" if all(qualification_checks.values()) else "REVIEW" summary = { @@ -719,7 +801,9 @@ def confirmation_arm( def main() -> None: - parser = argparse.ArgumentParser(description="Final untouched Fashionpedia validation confirmation.") + parser = argparse.ArgumentParser( + description="Final untouched Fashionpedia validation confirmation." + ) parser.add_argument("--run-root", required=True) parser.add_argument("--source-train-manifest", required=True) parser.add_argument("--source-dev-manifest", required=True) @@ -767,11 +851,15 @@ def main() -> None: if run_summary.get(arm_name, {}).get("status") != "PASS": raise RuntimeError(f"Development arm is not PASS: {arm_name}") - target_ids = [int(value) for value in config.get("target_attribute_ids", TARGET_IDS_DEFAULT)] + target_ids = [ + int(value) for value in config.get("target_attribute_ids", TARGET_IDS_DEFAULT) + ] if target_ids != TARGET_IDS_DEFAULT: raise RuntimeError(f"Unexpected target class list: {target_ids}") target_set = set(target_ids) - target_names = {int(key): str(value) for key, value in config["target_attribute_names"].items()} + target_names = { + int(key): str(value) for key, value in config["target_attribute_names"].items() + } if set(target_names) != target_set: raise RuntimeError("Target attribute name map does not match target IDs.") padding_fraction = float(config["crop"]["padding_fraction"]) @@ -785,9 +873,13 @@ def main() -> None: required_columns(train_manifest, source_required, train_manifest_path) required_columns(dev_manifest, source_required, dev_manifest_path) source_rows = train_manifest + dev_manifest - source_pairs = [canonical_pair(row["annotation_id"], row["attribute_id"]) for row in source_rows] + source_pairs = [ + canonical_pair(row["annotation_id"], row["attribute_id"]) for row in source_rows + ] if len(source_pairs) != len(set(source_pairs)): - raise RuntimeError("Source train/dev manifests contain duplicate annotation/attribute pairs.") + raise RuntimeError( + "Source train/dev manifests contain duplicate annotation/attribute pairs." + ) train_payload = load_json(train_annotation_path) reconstructed_pairs = build_train_rule_pairs(train_payload, target_set) @@ -826,9 +918,18 @@ def main() -> None: class_counts = Counter(int(row["attribute_id"]) for row in validation_rows) missing_classes = sorted(target_set - set(class_counts)) if missing_classes: - raise RuntimeError(f"Official validation lacks target classes: {missing_classes}") - if len({canonical_pair(row["annotation_id"], row["attribute_id"]) for row in validation_rows}) != len(validation_rows): - raise RuntimeError("Official validation manifest contains duplicate annotation/attribute pairs.") + raise RuntimeError( + f"Official validation lacks target classes: {missing_classes}" + ) + if len( + { + canonical_pair(row["annotation_id"], row["attribute_id"]) + for row in validation_rows + } + ) != len(validation_rows): + raise RuntimeError( + "Official validation manifest contains duplicate annotation/attribute pairs." + ) manifest_path = manifest_dir / "official_validation_manifest.csv" write_csv( @@ -847,7 +948,10 @@ def main() -> None: }, "official_validation": { **val_stats, - "class_counts": {str(attribute_id): int(class_counts[attribute_id]) for attribute_id in target_ids}, + "class_counts": { + str(attribute_id): int(class_counts[attribute_id]) + for attribute_id in target_ids + }, "source_train_dev_image_id_overlap_count": len(image_overlap), "all_images_resolved": True, }, @@ -869,7 +973,9 @@ def main() -> None: }, }, } - write_json(manifest_dir / "official_validation_manifest_summary.json", manifest_summary) + write_json( + manifest_dir / "official_validation_manifest_summary.json", manifest_summary + ) print("===== FINAL CONFIRMATION MANIFEST AUDIT =====", flush=True) print("STATUS: PASS", flush=True) @@ -880,14 +986,21 @@ def main() -> None: print(f"OFFICIAL_VALIDATION_MANIFEST: {manifest_path}", flush=True) labels = np.asarray( - [{attribute_id: index for index, attribute_id in enumerate(target_ids)}[int(row["attribute_id"])] for row in validation_rows], + [ + {attribute_id: index for index, attribute_id in enumerate(target_ids)}[ + int(row["attribute_id"]) + ] + for row in validation_rows + ], dtype=np.int64, ) cache_dir.mkdir(parents=True, exist_ok=True) device = torch.device("cuda") torch.cuda.empty_cache() torch.cuda.reset_peak_memory_stats(device) - image_processor = AutoImageProcessor.from_pretrained(model_id, cache_dir=str(cache_dir), use_fast=False) + image_processor = AutoImageProcessor.from_pretrained( + model_id, cache_dir=str(cache_dir), use_fast=False + ) frozen_summary = confirmation_arm( "frozen_head", @@ -927,7 +1040,11 @@ def main() -> None: comparisons: dict[str, dict[str, float]] = {} for scope, frozen_metrics, lora_metrics in ( ("raw", frozen_summary["raw_metrics"], lora_summary["raw_metrics"]), - ("temperature_calibrated", frozen_summary["temperature_calibrated_metrics"], lora_summary["temperature_calibrated_metrics"]), + ( + "temperature_calibrated", + frozen_summary["temperature_calibrated_metrics"], + lora_summary["temperature_calibrated_metrics"], + ), ): comparisons[scope] = { metric: float(lora_metrics[metric] - frozen_metrics[metric]) @@ -941,27 +1058,34 @@ def main() -> None: "official_validation_all_target_classes_present": len(missing_classes) == 0, "frozen_confirmation_pass": frozen_summary["status"] == "PASS", "lora_confirmation_pass": lora_summary["status"] == "PASS", - "lora_adapter_parameter_count_matches_development": lora_summary["qualification_checks"].get("adapter_parameter_count_matches_development", False), + "lora_adapter_parameter_count_matches_development": lora_summary[ + "qualification_checks" + ].get("adapter_parameter_count_matches_development", False), } overall_status = "PASS" if all(qualification_checks.values()) else "REVIEW" summary = { "status": overall_status, "schema_version": "fashionpedia_neckline_final_confirmation_v1", - "created_utc": datetime.now(timezone.utc).isoformat(), + "created_utc": datetime.now(UTC).isoformat(), "evaluation_scope": "untouched_official_fashionpedia_validation", "selection_scope": "fixed_from_train_only_development_before_validation_access", "model_id": model_id, "data": { "official_validation_samples": len(validation_rows), "official_validation_images": len(confirmation_ids), - "class_counts": {str(attribute_id): int(class_counts[attribute_id]) for attribute_id in target_ids}, + "class_counts": { + str(attribute_id): int(class_counts[attribute_id]) + for attribute_id in target_ids + }, }, "manifest_summary": manifest_summary, "frozen_head": frozen_summary, "lora": lora_summary, "lora_minus_frozen_head": comparisons, "qualification_checks": qualification_checks, - "gpu_peak_memory_allocated_mib": round(torch.cuda.max_memory_allocated(device) / (1024 ** 2), 2), + "gpu_peak_memory_allocated_mib": round( + torch.cuda.max_memory_allocated(device) / (1024**2), 2 + ), "elapsed_seconds": round(time.perf_counter() - started, 3), "claim_boundary": "External confirmation is limited to Fashionpedia validation instances eligible under the source task rule. No post-validation checkpoint, hyperparameter, crop, rank, or target-module changes were performed by this runner.", } @@ -970,13 +1094,32 @@ def main() -> None: print("===== FINAL FASHIONPEDIA CONFIRMATION SUMMARY =====", flush=True) print(f"STATUS: {overall_status}", flush=True) - print(f"FROZEN_RAW_MACRO_F1: {frozen_summary['raw_metrics']['macro_f1']:.6f}", flush=True) - print(f"LORA_RAW_MACRO_F1: {lora_summary['raw_metrics']['macro_f1']:.6f}", flush=True) - print(f"LORA_MINUS_FROZEN_RAW_MACRO_F1: {comparisons['raw']['macro_f1']:.6f}", flush=True) - print(f"FROZEN_RAW_ECE: {frozen_summary['raw_metrics']['top_label_ece']:.6f}", flush=True) - print(f"LORA_RAW_ECE: {lora_summary['raw_metrics']['top_label_ece']:.6f}", flush=True) - print(f"FROZEN_CALIBRATED_ECE: {frozen_summary['temperature_calibrated_metrics']['top_label_ece']:.6f}", flush=True) - print(f"LORA_CALIBRATED_ECE: {lora_summary['temperature_calibrated_metrics']['top_label_ece']:.6f}", flush=True) + print( + f"FROZEN_RAW_MACRO_F1: {frozen_summary['raw_metrics']['macro_f1']:.6f}", + flush=True, + ) + print( + f"LORA_RAW_MACRO_F1: {lora_summary['raw_metrics']['macro_f1']:.6f}", flush=True + ) + print( + f"LORA_MINUS_FROZEN_RAW_MACRO_F1: {comparisons['raw']['macro_f1']:.6f}", + flush=True, + ) + print( + f"FROZEN_RAW_ECE: {frozen_summary['raw_metrics']['top_label_ece']:.6f}", + flush=True, + ) + print( + f"LORA_RAW_ECE: {lora_summary['raw_metrics']['top_label_ece']:.6f}", flush=True + ) + print( + f"FROZEN_CALIBRATED_ECE: {frozen_summary['temperature_calibrated_metrics']['top_label_ece']:.6f}", + flush=True, + ) + print( + f"LORA_CALIBRATED_ECE: {lora_summary['temperature_calibrated_metrics']['top_label_ece']:.6f}", + flush=True, + ) print(f"SUMMARY_JSON: {summary_path}", flush=True) if overall_status != "PASS": raise SystemExit(2) diff --git a/scripts/final_confirmation_fashionpedia_neckline_v2.py b/scripts/final_confirmation_fashionpedia_neckline_v2.py index 2427b85..147958c 100644 --- a/scripts/final_confirmation_fashionpedia_neckline_v2.py +++ b/scripts/final_confirmation_fashionpedia_neckline_v2.py @@ -10,17 +10,19 @@ import sys import time from collections import Counter -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any import numpy as np +import peft import scipy -from scipy.optimize import minimize_scalar import sklearn import torch -from PIL import Image +import transformers from peft import PeftModel +from PIL import Image +from scipy.optimize import minimize_scalar from sklearn.metrics import ( accuracy_score, balanced_accuracy_score, @@ -30,9 +32,6 @@ precision_recall_fscore_support, ) from transformers import AutoImageProcessor, AutoModel -import peft -import transformers - EPS = 1e-12 TARGET_IDS_DEFAULT = [182, 183, 185, 187, 189, 190, 200] @@ -69,7 +68,9 @@ def write_csv(path: Path, fieldnames: list[str], rows: list[dict[str, Any]]) -> writer.writerows(rows) -def required_columns(rows: list[dict[str, str]], required: set[str], artifact: Path) -> None: +def required_columns( + rows: list[dict[str, str]], required: set[str], artifact: Path +) -> None: missing = sorted(required - set(rows[0])) if missing: raise RuntimeError(f"{artifact} is missing required columns: {missing}") @@ -308,7 +309,9 @@ def crop_from_record(record: dict[str, Any], padding_fraction: float) -> Image.I width = float(record["bbox_width"]) height = float(record["bbox_height"]) if width <= 0.0 or height <= 0.0: - raise ValueError(f"Non-positive bbox for annotation_id={record['annotation_id']}") + raise ValueError( + f"Non-positive bbox for annotation_id={record['annotation_id']}" + ) with Image.open(image_path) as source: image = source.convert("RGB") image_width, image_height = image.size @@ -319,7 +322,9 @@ def crop_from_record(record: dict[str, Any], padding_fraction: float) -> Image.I right = min(image_width, int(math.ceil(x + width + pad_x))) bottom = min(image_height, int(math.ceil(y + height + pad_y))) if right <= left or bottom <= top: - raise ValueError(f"Degenerate crop for annotation_id={record['annotation_id']}") + raise ValueError( + f"Degenerate crop for annotation_id={record['annotation_id']}" + ) return image.crop((left, top, right, bottom)) @@ -347,7 +352,9 @@ def softmax(logits: np.ndarray) -> np.ndarray: return probabilities -def top_label_ece(probabilities: np.ndarray, targets: np.ndarray, bins: int) -> tuple[float, list[dict[str, Any]]]: +def top_label_ece( + probabilities: np.ndarray, targets: np.ndarray, bins: int +) -> tuple[float, list[dict[str, Any]]]: predictions = probabilities.argmax(axis=1) confidence = probabilities.max(axis=1) correct = (predictions == targets).astype(np.float64) @@ -393,7 +400,9 @@ def metric_bundle( accuracy = float(accuracy_score(labels, predictions)) balanced_accuracy = float(balanced_accuracy_score(labels, predictions)) macro_f1 = float(f1_score(labels, predictions, average="macro", zero_division=0)) - weighted_f1 = float(f1_score(labels, predictions, average="weighted", zero_division=0)) + weighted_f1 = float( + f1_score(labels, predictions, average="weighted", zero_division=0) + ) nll = float(log_loss(labels, probabilities, labels=list(range(n_classes)))) one_hot = np.eye(n_classes, dtype=np.float64)[labels] brier = float(np.mean(np.sum((probabilities - one_hot) ** 2, axis=1))) @@ -439,12 +448,21 @@ def probabilities_from_saved_dev_predictions( target_ids: list[int], ) -> tuple[np.ndarray, np.ndarray]: rows = load_csv(path) - required = {"true_attribute_id"} | {f"probability_attribute_{attr_id}" for attr_id in target_ids} + required = {"true_attribute_id"} | { + f"probability_attribute_{attr_id}" for attr_id in target_ids + } required_columns(rows, required, path) - target_to_index = {attribute_id: index for index, attribute_id in enumerate(target_ids)} - labels = np.asarray([target_to_index[int(row["true_attribute_id"])] for row in rows], dtype=np.int64) + target_to_index = { + attribute_id: index for index, attribute_id in enumerate(target_ids) + } + labels = np.asarray( + [target_to_index[int(row["true_attribute_id"])] for row in rows], dtype=np.int64 + ) probabilities = np.asarray( - [[float(row[f"probability_attribute_{attr_id}"]) for attr_id in target_ids] for row in rows], + [ + [float(row[f"probability_attribute_{attr_id}"]) for attr_id in target_ids] + for row in rows + ], dtype=np.float64, ) if not np.isfinite(probabilities).all() or np.any(probabilities < 0.0): @@ -458,13 +476,19 @@ def probabilities_from_saved_dev_predictions( def fit_temperature(probabilities: np.ndarray, labels: np.ndarray) -> dict[str, Any]: base_logits = np.log(np.clip(probabilities, EPS, 1.0)) - before = float(log_loss(labels, softmax(base_logits), labels=list(range(probabilities.shape[1])))) + before = float( + log_loss( + labels, softmax(base_logits), labels=list(range(probabilities.shape[1])) + ) + ) def objective(temperature: float) -> float: if not np.isfinite(temperature) or temperature <= 0.0: return float("inf") calibrated = softmax(base_logits / float(temperature)) - return float(log_loss(labels, calibrated, labels=list(range(probabilities.shape[1])))) + return float( + log_loss(labels, calibrated, labels=list(range(probabilities.shape[1]))) + ) result = minimize_scalar( objective, @@ -477,7 +501,9 @@ def objective(temperature: float) -> float: temperature = float(result.x) after = objective(temperature) calibrated = softmax(base_logits / temperature) - predictions_unchanged = bool(np.array_equal(probabilities.argmax(axis=1), calibrated.argmax(axis=1))) + predictions_unchanged = bool( + np.array_equal(probabilities.argmax(axis=1), calibrated.argmax(axis=1)) + ) return { "method": "single_scalar_temperature_scaling", "optimization_scope": "selected_development_predictions_only", @@ -511,7 +537,9 @@ def predict_logits( chunks: list[np.ndarray] = [] for start in range(0, len(records), batch_size): end = min(start + batch_size, len(records)) - inputs = make_inputs(records[start:end], image_processor, device, padding_fraction) + inputs = make_inputs( + records[start:end], image_processor, device, padding_fraction + ) with torch.inference_mode(): with torch.autocast(device_type="cuda", dtype=torch.bfloat16): features = encoder.get_image_features(**inputs) @@ -527,7 +555,9 @@ def predict_logits( return logits -def load_head(checkpoint_path: Path, input_dimension: int, class_count: int, device: torch.device) -> torch.nn.Module: +def load_head( + checkpoint_path: Path, input_dimension: int, class_count: int, device: torch.device +) -> torch.nn.Module: checkpoint = torch.load(checkpoint_path, map_location="cpu") if "head_state_dict" not in checkpoint: raise RuntimeError(f"Missing head_state_dict in {checkpoint_path}") @@ -561,11 +591,15 @@ def load_frozen_arm( for parameter in model.parameters(): parameter.requires_grad = False head = load_head(checkpoint_path, input_dimension, class_count, device) - return model, head, { - "model_class": model.__class__.__name__, - "adapter_parameter_count": 0, - "checkpoint_path": str(checkpoint_path), - } + return ( + model, + head, + { + "model_class": model.__class__.__name__, + "adapter_parameter_count": 0, + "checkpoint_path": str(checkpoint_path), + }, + ) def load_lora_arm( @@ -590,13 +624,17 @@ def load_lora_arm( parameter.requires_grad = False encoder = lora_model.get_base_model() head = load_head(checkpoint_path, input_dimension, class_count, device) - return encoder, head, { - "model_class": encoder.__class__.__name__, - "adapter_model_class": lora_model.__class__.__name__, - "adapter_parameter_count": count_lora_parameters(lora_model), - "checkpoint_path": str(checkpoint_path), - "adapter_directory": str(adapter_dir), - } + return ( + encoder, + head, + { + "model_class": encoder.__class__.__name__, + "adapter_model_class": lora_model.__class__.__name__, + "adapter_parameter_count": count_lora_parameters(lora_model), + "checkpoint_path": str(checkpoint_path), + "adapter_directory": str(adapter_dir), + }, + ) def prediction_rows( @@ -625,13 +663,19 @@ def prediction_rows( "raw_confidence": float(raw_probabilities[index].max()), "raw_correct": bool(raw_indices[index] == labels[index]), "calibrated_predicted_attribute_id": calibrated_attribute_id, - "calibrated_predicted_attribute_name": target_names[calibrated_attribute_id], + "calibrated_predicted_attribute_name": target_names[ + calibrated_attribute_id + ], "calibrated_confidence": float(calibrated_probabilities[index].max()), "calibrated_correct": bool(calibrated_indices[index] == labels[index]), } for class_index, attribute_id in enumerate(target_ids): - row[f"raw_probability_attribute_{attribute_id}"] = float(raw_probabilities[index, class_index]) - row[f"calibrated_probability_attribute_{attribute_id}"] = float(calibrated_probabilities[index, class_index]) + row[f"raw_probability_attribute_{attribute_id}"] = float( + raw_probabilities[index, class_index] + ) + row[f"calibrated_probability_attribute_{attribute_id}"] = float( + calibrated_probabilities[index, class_index] + ) rows.append(row) return rows @@ -659,9 +703,13 @@ def confirmation_arm( head_path = Path(str(artifacts.get("head_state_path", ""))) dev_predictions_path = Path(str(artifacts.get("best_dev_predictions_csv", ""))) if not head_path.is_file(): - raise FileNotFoundError(f"Selected head checkpoint is missing for {arm_name}: {head_path}") + raise FileNotFoundError( + f"Selected head checkpoint is missing for {arm_name}: {head_path}" + ) if not dev_predictions_path.is_file(): - raise FileNotFoundError(f"Selected development predictions are missing for {arm_name}: {dev_predictions_path}") + raise FileNotFoundError( + f"Selected development predictions are missing for {arm_name}: {dev_predictions_path}" + ) input_dimension = 768 class_count = len(target_ids) @@ -674,12 +722,20 @@ def confirmation_arm( if not adapter_dir.is_dir(): raise FileNotFoundError(f"Selected LoRA adapter is missing: {adapter_dir}") encoder, head, load_info = load_lora_arm( - model_id, cache_dir, adapter_dir, head_path, input_dimension, class_count, device + model_id, + cache_dir, + adapter_dir, + head_path, + input_dimension, + class_count, + device, ) else: raise ValueError(f"Unknown arm: {arm_name}") - dev_probabilities, dev_labels = probabilities_from_saved_dev_predictions(dev_predictions_path, target_ids) + dev_probabilities, dev_labels = probabilities_from_saved_dev_predictions( + dev_predictions_path, target_ids + ) temperature_info = fit_temperature(dev_probabilities, dev_labels) temperature = float(temperature_info["temperature"]) @@ -698,12 +754,21 @@ def confirmation_arm( raw_prediction_indices = raw_probabilities.argmax(axis=1) calibrated_prediction_indices = calibrated_probabilities.argmax(axis=1) - calibration_argmax_unchanged = bool(np.array_equal(raw_prediction_indices, calibrated_prediction_indices)) + calibration_argmax_unchanged = bool( + np.array_equal(raw_prediction_indices, calibrated_prediction_indices) + ) if not calibration_argmax_unchanged: - raise RuntimeError("Temperature scaling unexpectedly changed final argmax predictions.") + raise RuntimeError( + "Temperature scaling unexpectedly changed final argmax predictions." + ) rows = prediction_rows( - records, labels, raw_probabilities, calibrated_probabilities, target_ids, target_names + records, + labels, + raw_probabilities, + calibrated_probabilities, + target_ids, + target_names, ) prediction_fields = list(rows[0].keys()) predictions_path = arm_dir / "confirmation_predictions.csv" @@ -720,8 +785,22 @@ def confirmation_arm( ["attribute_id", "attribute_name", "precision", "recall", "f1", "support"], per_class, ) - write_json(raw_ece_path, {"top_label_ece": raw_metrics["top_label_ece"], "bin_count": ece_bins, "bins": raw_ece_rows}) - write_json(calibrated_ece_path, {"top_label_ece": calibrated_metrics["top_label_ece"], "bin_count": ece_bins, "bins": calibrated_ece_rows}) + write_json( + raw_ece_path, + { + "top_label_ece": raw_metrics["top_label_ece"], + "bin_count": ece_bins, + "bins": raw_ece_rows, + }, + ) + write_json( + calibrated_ece_path, + { + "top_label_ece": calibrated_metrics["top_label_ece"], + "bin_count": ece_bins, + "bins": calibrated_ece_rows, + }, + ) write_json(temperature_path, temperature_info) np.save(logits_path, raw_logits.astype(np.float32, copy=False)) @@ -732,16 +811,19 @@ def confirmation_arm( "raw_prediction_count_matches_confirmation": len(rows) == len(records), "temperature_optimizer_success": bool(temperature_info["optimizer_success"]), "temperature_positive": temperature > 0.0, - "temperature_preserves_argmax_on_development": bool(temperature_info["development_argmax_unchanged"]), + "temperature_preserves_argmax_on_development": bool( + temperature_info["development_argmax_unchanged"] + ), "temperature_preserves_argmax_on_confirmation": calibration_argmax_unchanged, "finite_raw_metrics": all(np.isfinite(value) for value in raw_metrics.values()), - "finite_calibrated_metrics": all(np.isfinite(value) for value in calibrated_metrics.values()), + "finite_calibrated_metrics": all( + np.isfinite(value) for value in calibrated_metrics.values() + ), } if arm_name == "lora": - qualification_checks["adapter_parameter_count_matches_development"] = ( - int(load_info["adapter_parameter_count"]) - == int(development_arm_summary["parameter_counts"]["lora_adapter"]) - ) + qualification_checks["adapter_parameter_count_matches_development"] = int( + load_info["adapter_parameter_count"] + ) == int(development_arm_summary["parameter_counts"]["lora_adapter"]) status = "PASS" if all(qualification_checks.values()) else "REVIEW" summary = { @@ -781,7 +863,9 @@ def confirmation_arm( def main() -> None: - parser = argparse.ArgumentParser(description="Final untouched Fashionpedia validation confirmation.") + parser = argparse.ArgumentParser( + description="Final untouched Fashionpedia validation confirmation." + ) parser.add_argument("--run-root", required=True) parser.add_argument("--source-train-manifest", required=True) parser.add_argument("--source-dev-manifest", required=True) @@ -832,11 +916,15 @@ def main() -> None: if run_summary.get(arm_name, {}).get("status") != "PASS": raise RuntimeError(f"Development arm is not PASS: {arm_name}") - target_ids = [int(value) for value in config.get("target_attribute_ids", TARGET_IDS_DEFAULT)] + target_ids = [ + int(value) for value in config.get("target_attribute_ids", TARGET_IDS_DEFAULT) + ] if target_ids != TARGET_IDS_DEFAULT: raise RuntimeError(f"Unexpected target class list: {target_ids}") target_set = set(target_ids) - target_names = {int(key): str(value) for key, value in config["target_attribute_names"].items()} + target_names = { + int(key): str(value) for key, value in config["target_attribute_names"].items() + } if set(target_names) != target_set: raise RuntimeError("Target attribute name map does not match target IDs.") @@ -858,9 +946,13 @@ def main() -> None: required_columns(train_manifest, source_required, train_manifest_path) required_columns(dev_manifest, source_required, dev_manifest_path) source_rows = train_manifest + dev_manifest - source_pairs = [canonical_pair(row["annotation_id"], row["attribute_id"]) for row in source_rows] + source_pairs = [ + canonical_pair(row["annotation_id"], row["attribute_id"]) for row in source_rows + ] if len(source_pairs) != len(set(source_pairs)): - raise RuntimeError("Source train/dev manifests contain duplicate annotation/attribute pairs.") + raise RuntimeError( + "Source train/dev manifests contain duplicate annotation/attribute pairs." + ) train_payload = load_json(train_annotation_path) reconstructed_pairs = build_train_rule_pairs( @@ -904,9 +996,18 @@ def main() -> None: class_counts = Counter(int(row["attribute_id"]) for row in validation_rows) missing_classes = sorted(target_set - set(class_counts)) if missing_classes: - raise RuntimeError(f"Official validation lacks target classes: {missing_classes}") - if len({canonical_pair(row["annotation_id"], row["attribute_id"]) for row in validation_rows}) != len(validation_rows): - raise RuntimeError("Official validation manifest contains duplicate annotation/attribute pairs.") + raise RuntimeError( + f"Official validation lacks target classes: {missing_classes}" + ) + if len( + { + canonical_pair(row["annotation_id"], row["attribute_id"]) + for row in validation_rows + } + ) != len(validation_rows): + raise RuntimeError( + "Official validation manifest contains duplicate annotation/attribute pairs." + ) manifest_path = manifest_dir / "official_validation_manifest.csv" write_csv( @@ -938,7 +1039,10 @@ def main() -> None: }, "official_validation": { **val_stats, - "class_counts": {str(attribute_id): int(class_counts[attribute_id]) for attribute_id in target_ids}, + "class_counts": { + str(attribute_id): int(class_counts[attribute_id]) + for attribute_id in target_ids + }, "source_train_dev_image_id_overlap_count": len(image_overlap), "all_images_resolved": True, }, @@ -962,7 +1066,9 @@ def main() -> None: }, }, } - write_json(manifest_dir / "official_validation_manifest_summary.json", manifest_summary) + write_json( + manifest_dir / "official_validation_manifest_summary.json", manifest_summary + ) print("===== FINAL CONFIRMATION MANIFEST AUDIT =====", flush=True) print("STATUS: PASS", flush=True) @@ -977,14 +1083,21 @@ def main() -> None: print(f"OFFICIAL_VALIDATION_MANIFEST: {manifest_path}", flush=True) labels = np.asarray( - [{attribute_id: index for index, attribute_id in enumerate(target_ids)}[int(row["attribute_id"])] for row in validation_rows], + [ + {attribute_id: index for index, attribute_id in enumerate(target_ids)}[ + int(row["attribute_id"]) + ] + for row in validation_rows + ], dtype=np.int64, ) cache_dir.mkdir(parents=True, exist_ok=True) device = torch.device("cuda") torch.cuda.empty_cache() torch.cuda.reset_peak_memory_stats(device) - image_processor = AutoImageProcessor.from_pretrained(model_id, cache_dir=str(cache_dir), use_fast=False) + image_processor = AutoImageProcessor.from_pretrained( + model_id, cache_dir=str(cache_dir), use_fast=False + ) frozen_summary = confirmation_arm( "frozen_head", @@ -1024,7 +1137,11 @@ def main() -> None: comparisons: dict[str, dict[str, float]] = {} for scope, frozen_metrics, lora_metrics in ( ("raw", frozen_summary["raw_metrics"], lora_summary["raw_metrics"]), - ("temperature_calibrated", frozen_summary["temperature_calibrated_metrics"], lora_summary["temperature_calibrated_metrics"]), + ( + "temperature_calibrated", + frozen_summary["temperature_calibrated_metrics"], + lora_summary["temperature_calibrated_metrics"], + ), ): comparisons[scope] = { metric: float(lora_metrics[metric] - frozen_metrics[metric]) @@ -1038,13 +1155,15 @@ def main() -> None: "official_validation_all_target_classes_present": len(missing_classes) == 0, "frozen_confirmation_pass": frozen_summary["status"] == "PASS", "lora_confirmation_pass": lora_summary["status"] == "PASS", - "lora_adapter_parameter_count_matches_development": lora_summary["qualification_checks"].get("adapter_parameter_count_matches_development", False), + "lora_adapter_parameter_count_matches_development": lora_summary[ + "qualification_checks" + ].get("adapter_parameter_count_matches_development", False), } overall_status = "PASS" if all(qualification_checks.values()) else "REVIEW" summary = { "status": overall_status, "schema_version": "fashionpedia_neckline_final_confirmation_v2", - "created_utc": datetime.now(timezone.utc).isoformat(), + "created_utc": datetime.now(UTC).isoformat(), "evaluation_scope": "untouched_official_fashionpedia_validation", "selection_scope": "fixed_from_train_only_development_before_validation_access", "model_id": model_id, @@ -1053,14 +1172,19 @@ def main() -> None: "official_validation_images": len(confirmation_ids), "target_category_id": target_category_id, "target_category_name": target_category_name, - "class_counts": {str(attribute_id): int(class_counts[attribute_id]) for attribute_id in target_ids}, + "class_counts": { + str(attribute_id): int(class_counts[attribute_id]) + for attribute_id in target_ids + }, }, "manifest_summary": manifest_summary, "frozen_head": frozen_summary, "lora": lora_summary, "lora_minus_frozen_head": comparisons, "qualification_checks": qualification_checks, - "gpu_peak_memory_allocated_mib": round(torch.cuda.max_memory_allocated(device) / (1024 ** 2), 2), + "gpu_peak_memory_allocated_mib": round( + torch.cuda.max_memory_allocated(device) / (1024**2), 2 + ), "elapsed_seconds": round(time.perf_counter() - started, 3), "claim_boundary": "External confirmation is limited to Fashionpedia validation instances eligible under the source task rule. No post-validation checkpoint, hyperparameter, crop, rank, or target-module changes were performed by this runner.", } @@ -1069,13 +1193,32 @@ def main() -> None: print("===== FINAL FASHIONPEDIA CONFIRMATION SUMMARY =====", flush=True) print(f"STATUS: {overall_status}", flush=True) - print(f"FROZEN_RAW_MACRO_F1: {frozen_summary['raw_metrics']['macro_f1']:.6f}", flush=True) - print(f"LORA_RAW_MACRO_F1: {lora_summary['raw_metrics']['macro_f1']:.6f}", flush=True) - print(f"LORA_MINUS_FROZEN_RAW_MACRO_F1: {comparisons['raw']['macro_f1']:.6f}", flush=True) - print(f"FROZEN_RAW_ECE: {frozen_summary['raw_metrics']['top_label_ece']:.6f}", flush=True) - print(f"LORA_RAW_ECE: {lora_summary['raw_metrics']['top_label_ece']:.6f}", flush=True) - print(f"FROZEN_CALIBRATED_ECE: {frozen_summary['temperature_calibrated_metrics']['top_label_ece']:.6f}", flush=True) - print(f"LORA_CALIBRATED_ECE: {lora_summary['temperature_calibrated_metrics']['top_label_ece']:.6f}", flush=True) + print( + f"FROZEN_RAW_MACRO_F1: {frozen_summary['raw_metrics']['macro_f1']:.6f}", + flush=True, + ) + print( + f"LORA_RAW_MACRO_F1: {lora_summary['raw_metrics']['macro_f1']:.6f}", flush=True + ) + print( + f"LORA_MINUS_FROZEN_RAW_MACRO_F1: {comparisons['raw']['macro_f1']:.6f}", + flush=True, + ) + print( + f"FROZEN_RAW_ECE: {frozen_summary['raw_metrics']['top_label_ece']:.6f}", + flush=True, + ) + print( + f"LORA_RAW_ECE: {lora_summary['raw_metrics']['top_label_ece']:.6f}", flush=True + ) + print( + f"FROZEN_CALIBRATED_ECE: {frozen_summary['temperature_calibrated_metrics']['top_label_ece']:.6f}", + flush=True, + ) + print( + f"LORA_CALIBRATED_ECE: {lora_summary['temperature_calibrated_metrics']['top_label_ece']:.6f}", + flush=True, + ) print(f"SUMMARY_JSON: {summary_path}", flush=True) if overall_status != "PASS": raise SystemExit(2) diff --git a/scripts/preflight_siglip2_bf16_one_update.py b/scripts/preflight_siglip2_bf16_one_update.py index 6a4184c..40d6237 100644 --- a/scripts/preflight_siglip2_bf16_one_update.py +++ b/scripts/preflight_siglip2_bf16_one_update.py @@ -15,11 +15,10 @@ import torch import torch.nn.functional as F import transformers -from PIL import Image from peft import LoraConfig, get_peft_model +from PIL import Image from transformers import AutoImageProcessor, AutoModel - ENCODER_SELF_ATTN_PATTERN = re.compile( r"^vision_model\.encoder\.layers\.\d+\.self_attn\." r"(q_proj|k_proj|v_proj|out_proj)$" @@ -69,8 +68,7 @@ def crop_from_record( if width <= 0.0 or height <= 0.0: raise ValueError( - "Non-positive bbox for annotation_id=" - f"{record['annotation_id']}" + f"Non-positive bbox for annotation_id={record['annotation_id']}" ) with Image.open(image_path) as source: @@ -93,8 +91,7 @@ def crop_from_record( if right <= left or bottom <= top: raise ValueError( - "Degenerate crop for annotation_id=" - f"{record['annotation_id']}" + f"Degenerate crop for annotation_id={record['annotation_id']}" ) return image.crop((left, top, right, bottom)) @@ -106,10 +103,7 @@ def make_inputs( device: torch.device, padding_fraction: float, ) -> dict[str, torch.Tensor]: - crops = [ - crop_from_record(record, padding_fraction) - for record in records - ] + crops = [crop_from_record(record, padding_fraction) for record in records] processor_outputs = image_processor( images=crops, @@ -225,10 +219,7 @@ def make_class_weights( raise RuntimeError("At least one target class is absent.") total = float(len(labels)) - weights = total / ( - float(class_count) - * counts.astype(np.float64) - ) + weights = total / (float(class_count) * counts.astype(np.float64)) return torch.tensor( weights, @@ -313,9 +304,7 @@ def run_one_update( ) if not torch.isfinite(raw_loss).item(): - raise FloatingPointError( - f"Non-finite loss in arm={arm_name}" - ) + raise FloatingPointError(f"Non-finite loss in arm={arm_name}") (raw_loss / float(len(batches))).backward() losses.append(float(raw_loss.detach().cpu().item())) @@ -380,9 +369,7 @@ def main() -> None: raise RuntimeError("CUDA is unavailable.") if not torch.cuda.is_bf16_supported(): - raise RuntimeError( - "This CUDA device does not report bf16 support." - ) + raise RuntimeError("This CUDA device does not report bf16 support.") config = load_json(config_path) audit = load_json(audit_path) @@ -395,21 +382,14 @@ def main() -> None: seed = int(config["training"]["seed"]) set_seed(seed) - target_attribute_ids = [ - int(value) - for value in config["target_attribute_ids"] - ] + target_attribute_ids = [int(value) for value in config["target_attribute_ids"]] class_index = { - attribute_id: index - for index, attribute_id in enumerate(target_attribute_ids) + attribute_id: index for index, attribute_id in enumerate(target_attribute_ids) } all_labels = np.asarray( - [ - class_index[int(record["attribute_id"])] - for record in records - ], + [class_index[int(record["attribute_id"])] for record in records], dtype=np.int64, ) @@ -419,35 +399,23 @@ def main() -> None: device=device, ) - physical_batch_size = int( - config["training"]["physical_batch_size"] - ) + physical_batch_size = int(config["training"]["physical_batch_size"]) - accumulation_steps = int( - config["training"]["gradient_accumulation_steps"] - ) + accumulation_steps = int(config["training"]["gradient_accumulation_steps"]) - probe_sample_count = ( - physical_batch_size - * accumulation_steps - ) + probe_sample_count = physical_batch_size * accumulation_steps # Matches epoch=1 shuffle policy from the failed trainer. - order = np.random.default_rng( - seed + 1 - ).permutation(len(records))[:probe_sample_count] - - probe_records = [ - records[int(index)] - for index in order + order = np.random.default_rng(seed + 1).permutation(len(records))[ + :probe_sample_count ] + probe_records = [records[int(index)] for index in order] + probe_labels = all_labels[order] batches = [ - probe_records[ - index:index + physical_batch_size - ] + probe_records[index : index + physical_batch_size] for index in range( 0, probe_sample_count, @@ -456,9 +424,7 @@ def main() -> None: ] batch_labels = [ - probe_labels[ - index:index + physical_batch_size - ] + probe_labels[index : index + physical_batch_size] for index in range( 0, probe_sample_count, @@ -466,9 +432,7 @@ def main() -> None: ) ] - padding_fraction = float( - config["crop"]["padding_fraction"] - ) + padding_fraction = float(config["crop"]["padding_fraction"]) image_processor = AutoImageProcessor.from_pretrained( args.model_id, @@ -476,41 +440,30 @@ def main() -> None: use_fast=False, ) - set_seed( - int(config["head"]["initialization_seed"]) - ) + set_seed(int(config["head"]["initialization_seed"])) template_head = torch.nn.Linear( int(config["head"]["input_dimension"]), int(config["head"]["output_classes"]), ) - initial_head_state = copy.deepcopy( - template_head.state_dict() - ) + initial_head_state = copy.deepcopy(template_head.state_dict()) del template_head - raw_targets = audit[ - "recommended_lora_scope" - ]["full_target_module_paths"] + raw_targets = audit["recommended_lora_scope"]["full_target_module_paths"] target_modules = sorted( module_name for module_name in raw_targets - if ENCODER_SELF_ATTN_PATTERN.fullmatch( - module_name - ) + if ENCODER_SELF_ATTN_PATTERN.fullmatch(module_name) ) - expected_targets = int( - config["lora"]["expected_target_module_count"] - ) + expected_targets = int(config["lora"]["expected_target_module_count"]) if len(target_modules) != expected_targets: raise RuntimeError( - "Unexpected LoRA target count: " - f"{len(target_modules)} vs {expected_targets}" + f"Unexpected LoRA target count: {len(target_modules)} vs {expected_targets}" ) torch.cuda.empty_cache() @@ -530,9 +483,7 @@ def main() -> None: int(config["head"]["output_classes"]), ).to(device) - frozen_head.load_state_dict( - copy.deepcopy(initial_head_state) - ) + frozen_head.load_state_dict(copy.deepcopy(initial_head_state)) frozen_optimizer = torch.optim.AdamW( frozen_head.parameters(), @@ -555,9 +506,7 @@ def main() -> None: ) frozen_encoder_grad_tensors = sum( - 1 - for parameter in frozen_model.parameters() - if parameter.grad is not None + 1 for parameter in frozen_model.parameters() if parameter.grad is not None ) del frozen_optimizer @@ -575,13 +524,8 @@ def main() -> None: module_map = dict(base_model.named_modules()) - if any( - target not in module_map - for target in target_modules - ): - raise RuntimeError( - "Audited LoRA target missing from base model." - ) + if any(target not in module_map for target in target_modules): + raise RuntimeError("Audited LoRA target missing from base model.") lora_config = LoraConfig( r=int(config["lora"]["rank"]), @@ -604,35 +548,23 @@ def main() -> None: int(config["head"]["output_classes"]), ).to(device) - lora_head.load_state_dict( - copy.deepcopy(initial_head_state) - ) + lora_head.load_state_dict(copy.deepcopy(initial_head_state)) adapter_parameters = [ - parameter - for parameter in lora_model.parameters() - if parameter.requires_grad + parameter for parameter in lora_model.parameters() if parameter.requires_grad ] lora_optimizer = torch.optim.AdamW( [ { "params": adapter_parameters, - "lr": float( - config["training"]["lora_learning_rate"] - ), - "weight_decay": float( - config["training"]["weight_decay"] - ), + "lr": float(config["training"]["lora_learning_rate"]), + "weight_decay": float(config["training"]["weight_decay"]), }, { "params": lora_head.parameters(), - "lr": float( - config["training"]["head_learning_rate"] - ), - "weight_decay": float( - config["training"]["weight_decay"] - ), + "lr": float(config["training"]["head_learning_rate"]), + "weight_decay": float(config["training"]["weight_decay"]), }, ] ) @@ -658,66 +590,43 @@ def main() -> None: unexpected_base_trainables = [ name for name, parameter in lora_model.named_parameters() - if parameter.requires_grad - and ".lora_A." not in name - and ".lora_B." not in name + if parameter.requires_grad and ".lora_A." not in name and ".lora_B." not in name ] torch.cuda.synchronize(device) expected_adapter_parameters = int( - config["lora"][ - "expected_adapter_parameter_count" - ] + config["lora"]["expected_adapter_parameter_count"] ) checks = { "cuda_available": bool(torch.cuda.is_available()), - "bf16_supported": bool( - torch.cuda.is_bf16_supported() - ), - "selected_lora_targets_match": ( - len(target_modules) == expected_targets - ), + "bf16_supported": bool(torch.cuda.is_bf16_supported()), + "selected_lora_targets_match": (len(target_modules) == expected_targets), "adapter_parameter_count_matches": ( - adapter_parameter_count - == expected_adapter_parameters + adapter_parameter_count == expected_adapter_parameters ), "frozen_gradients_finite_and_present": ( - frozen_result["gradients"][ - "all_finite_and_present" - ] + frozen_result["gradients"]["all_finite_and_present"] ), "frozen_optimizer_update_observed": ( - frozen_result["optimizer_update"][ - "optimizer_update_observed" - ] - ), - "frozen_encoder_has_no_grads": ( - frozen_encoder_grad_tensors == 0 + frozen_result["optimizer_update"]["optimizer_update_observed"] ), + "frozen_encoder_has_no_grads": (frozen_encoder_grad_tensors == 0), "lora_gradients_finite_and_present": ( - lora_result["gradients"][ - "all_finite_and_present" - ] + lora_result["gradients"]["all_finite_and_present"] ), "lora_optimizer_update_observed": ( - lora_result["optimizer_update"][ - "optimizer_update_observed" - ] - ), - "no_unexpected_base_trainables": ( - len(unexpected_base_trainables) == 0 + lora_result["optimizer_update"]["optimizer_update_observed"] ), + "no_unexpected_base_trainables": (len(unexpected_base_trainables) == 0), } status = "PASS" if all(checks.values()) else "REVIEW" payload = { "status": status, - "schema_version": ( - "siglip2_bf16_one_update_preflight_v1" - ), + "schema_version": ("siglip2_bf16_one_update_preflight_v1"), "precision_policy": { "autocast_dtype": "bfloat16", "grad_scaler_used": False, @@ -736,12 +645,9 @@ def main() -> None: "peft": peft.__version__, "device": str(device), "gpu_name": torch.cuda.get_device_name(0), - "bf16_supported": bool( - torch.cuda.is_bf16_supported() - ), + "bf16_supported": bool(torch.cuda.is_bf16_supported()), "peak_gpu_memory_mib": round( - torch.cuda.max_memory_allocated(device) - / (1024 ** 2), + torch.cuda.max_memory_allocated(device) / (1024**2), 2, ), }, @@ -749,24 +655,17 @@ def main() -> None: "physical_batch_size": physical_batch_size, "gradient_accumulation_steps": accumulation_steps, "effective_batch_size": probe_sample_count, - "annotation_ids": [ - record["annotation_id"] - for record in probe_records - ], + "annotation_ids": [record["annotation_id"] for record in probe_records], }, "frozen_head": { **frozen_result, - "encoder_gradient_tensor_count": ( - frozen_encoder_grad_tensors - ), + "encoder_gradient_tensor_count": (frozen_encoder_grad_tensors), }, "lora": { **lora_result, "selected_target_count": len(target_modules), "adapter_parameter_count": adapter_parameter_count, - "unexpected_base_trainables": ( - unexpected_base_trainables - ), + "unexpected_base_trainables": (unexpected_base_trainables), }, "checks": checks, "claim_boundary": ( @@ -783,10 +682,7 @@ def main() -> None: print("===== SIGLIP2 BF16 ONE-UPDATE PREFLIGHT =====") print(f"STATUS: {status}") - print( - "BF16_SUPPORTED: " - f"{payload['environment']['bf16_supported']}" - ) + print(f"BF16_SUPPORTED: {payload['environment']['bf16_supported']}") print( "FROZEN_GRADIENTS_FINITE_AND_PRESENT: " f"{checks['frozen_gradients_finite_and_present']}" @@ -799,10 +695,7 @@ def main() -> None: "LORA_GRADIENTS_FINITE_AND_PRESENT: " f"{checks['lora_gradients_finite_and_present']}" ) - print( - "LORA_OPTIMIZER_UPDATE_OBSERVED: " - f"{checks['lora_optimizer_update_observed']}" - ) + print(f"LORA_OPTIMIZER_UPDATE_OBSERVED: {checks['lora_optimizer_update_observed']}") print( "FROZEN_MISSING_GRADS: " f"{len(frozen_result['gradients']['missing_gradient_names'])}" @@ -812,8 +705,7 @@ def main() -> None: f"{len(frozen_result['gradients']['nonfinite_gradient_names'])}" ) print( - "LORA_MISSING_GRADS: " - f"{len(lora_result['gradients']['missing_gradient_names'])}" + f"LORA_MISSING_GRADS: {len(lora_result['gradients']['missing_gradient_names'])}" ) print( "LORA_NONFINITE_GRADS: " @@ -826,4 +718,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/preflight_siglip2_lora_matched_head.py b/scripts/preflight_siglip2_lora_matched_head.py index 055b2b5..bb896d4 100644 --- a/scripts/preflight_siglip2_lora_matched_head.py +++ b/scripts/preflight_siglip2_lora_matched_head.py @@ -14,11 +14,10 @@ import torch import torch.nn.functional as F import transformers -from PIL import Image from peft import LoraConfig, get_peft_model +from PIL import Image from transformers import AutoImageProcessor, AutoModel - ENCODER_SELF_ATTN_PATTERN = re.compile( r"^vision_model\.encoder\.layers\.\d+\.self_attn\." r"(q_proj|k_proj|v_proj|out_proj)$" @@ -79,8 +78,7 @@ def crop_from_record( if right <= left or bottom <= top: raise ValueError( - "Degenerate crop for annotation_id=" - f"{record['annotation_id']}" + f"Degenerate crop for annotation_id={record['annotation_id']}" ) return image.crop((left, top, right, bottom)) @@ -103,17 +101,14 @@ def trainable_parameter_count(module: torch.nn.Module) -> int: def finite_gradients(module: torch.nn.Module) -> bool: trainable = [ - parameter - for parameter in module.parameters() - if parameter.requires_grad + parameter for parameter in module.parameters() if parameter.requires_grad ] if not trainable: return False return all( - parameter.grad is not None - and torch.isfinite(parameter.grad).all().item() + parameter.grad is not None and torch.isfinite(parameter.grad).all().item() for parameter in trainable ) @@ -160,41 +155,26 @@ def main() -> None: device = torch.device("cuda") - expected_class_ids = [ - int(value) - for value in config["target_attribute_ids"] - ] + expected_class_ids = [int(value) for value in config["target_attribute_ids"]] class_index = { - attribute_id: index - for index, attribute_id in enumerate(expected_class_ids) + attribute_id: index for index, attribute_id in enumerate(expected_class_ids) } batch_size = int(config["preflight"]["batch_size"]) records = load_records(manifest_path, batch_size) - observed_class_ids = { - int(record["attribute_id"]) - for record in records - } + observed_class_ids = {int(record["attribute_id"]) for record in records} if not observed_class_ids.issubset(set(expected_class_ids)): raise RuntimeError("Preflight records contain unknown target labels.") - padding_fraction = float( - hog_config["crop"]["padding_fraction"] - ) + padding_fraction = float(hog_config["crop"]["padding_fraction"]) - crops = [ - crop_from_record(record, padding_fraction) - for record in records - ] + crops = [crop_from_record(record, padding_fraction) for record in records] labels = torch.tensor( - [ - class_index[int(record["attribute_id"])] - for record in records - ], + [class_index[int(record["attribute_id"])] for record in records], dtype=torch.long, device=device, ) @@ -216,23 +196,15 @@ def main() -> None: if hasattr(value, "to") } - raw_audit_targets = audit[ - "recommended_lora_scope" - ]["full_target_module_paths"] + raw_audit_targets = audit["recommended_lora_scope"]["full_target_module_paths"] selected_targets = sorted( - path - for path in raw_audit_targets - if ENCODER_SELF_ATTN_PATTERN.fullmatch(path) + path for path in raw_audit_targets if ENCODER_SELF_ATTN_PATTERN.fullmatch(path) ) - excluded_targets = sorted( - set(raw_audit_targets) - set(selected_targets) - ) + excluded_targets = sorted(set(raw_audit_targets) - set(selected_targets)) - expected_target_count = int( - config["lora"]["expected_target_module_count"] - ) + expected_target_count = int(config["lora"]["expected_target_module_count"]) if len(selected_targets) != expected_target_count: raise RuntimeError( @@ -243,10 +215,7 @@ def main() -> None: excluded_expected = str(config["lora"]["excluded_module"]) if excluded_targets != [excluded_expected]: - raise RuntimeError( - "Unexpected excluded LoRA target set: " - f"{excluded_targets}" - ) + raise RuntimeError(f"Unexpected excluded LoRA target set: {excluded_targets}") torch.manual_seed(int(config["head"]["initialization_seed"])) @@ -264,9 +233,7 @@ def main() -> None: int(config["head"]["output_classes"]), ).to(device) - frozen_head_initial_state = copy.deepcopy( - frozen_head.state_dict() - ) + frozen_head_initial_state = copy.deepcopy(frozen_head.state_dict()) with torch.no_grad(): frozen_features = frozen_model.get_image_features( @@ -282,9 +249,7 @@ def main() -> None: frozen_head_gradients_finite = finite_gradients(frozen_head) frozen_model_grad_count = sum( - 1 - for parameter in frozen_model.parameters() - if parameter.grad is not None + 1 for parameter in frozen_model.parameters() if parameter.grad is not None ) frozen_feature_shape = list(frozen_features.shape) @@ -308,15 +273,12 @@ def main() -> None: original_module_map = dict(base_model.named_modules()) missing_target_modules = [ - path - for path in selected_targets - if path not in original_module_map + path for path in selected_targets if path not in original_module_map ] if missing_target_modules: raise RuntimeError( - "Audited LoRA targets missing from model: " - f"{missing_target_modules}" + f"Audited LoRA targets missing from model: {missing_target_modules}" ) invalid_target_modules = [ @@ -330,8 +292,7 @@ def main() -> None: if invalid_target_modules: raise RuntimeError( - "Audited LoRA targets are not Linear modules: " - f"{invalid_target_modules}" + f"Audited LoRA targets are not Linear modules: {invalid_target_modules}" ) rank = int(config["lora"]["rank"]) @@ -387,10 +348,8 @@ def main() -> None: lora_head.load_state_dict(frozen_head_initial_state) with torch.no_grad(): - lora_features_initial = ( - lora_base_model.get_image_features( - **model_inputs, - ) + lora_features_initial = lora_base_model.get_image_features( + **model_inputs, ) lora_logits_initial = lora_head(lora_features_initial) @@ -409,10 +368,7 @@ def main() -> None: # would add memory pressure. Instead, use logits equality as the # adapter no-op qualification after matching head initialization. initial_logit_delta = float( - ( - lora_logits_initial.detach().float().cpu() - - frozen_logits_cpu - ) + (lora_logits_initial.detach().float().cpu() - frozen_logits_cpu) .abs() .max() .item() @@ -447,15 +403,11 @@ def main() -> None: unexpected_base_trainables = [ name for name, _ in lora_named_trainable - if ".lora_A." not in name - and ".lora_B." not in name + if ".lora_A." not in name and ".lora_B." not in name ] actual_lora_parameter_count = int( - sum( - parameter.numel() - for _, parameter in lora_adapter_trainables - ) + sum(parameter.numel() for _, parameter in lora_adapter_trainables) ) lora_gradients_finite = finite_gradients(lora_model) @@ -463,15 +415,10 @@ def main() -> None: torch.cuda.synchronize(device) - noop_tolerance = float( - config["preflight"]["initial_noop_max_abs_tolerance"] - ) + noop_tolerance = float(config["preflight"]["initial_noop_max_abs_tolerance"]) head_parameter_count = int( - sum( - parameter.numel() - for parameter in lora_head.parameters() - ) + sum(parameter.numel() for parameter in lora_head.parameters()) ) checks = { @@ -479,40 +426,21 @@ def main() -> None: "selected_target_count_matches": ( len(selected_targets) == expected_target_count ), - "excluded_pooling_head_only": ( - excluded_targets == [excluded_expected] - ), + "excluded_pooling_head_only": (excluded_targets == [excluded_expected]), "all_selected_targets_injected": ( len(injected_targets) == len(selected_targets) ), "adapter_parameter_count_matches": ( - actual_lora_parameter_count - == expected_lora_parameter_count - ), - "no_unexpected_base_trainables": ( - len(unexpected_base_trainables) == 0 - ), - "frozen_head_gradients_finite": ( - frozen_head_gradients_finite - ), - "frozen_encoder_has_no_grads": ( - frozen_model_grad_count == 0 - ), - "lora_adapter_gradients_finite": ( - lora_gradients_finite - ), - "lora_head_gradients_finite": ( - lora_head_gradients_finite - ), - "finite_lora_features": bool( - torch.isfinite(lora_features_train).all().item() - ), - "finite_lora_logits": bool( - torch.isfinite(lora_logits_train).all().item() - ), - "initial_logits_match_frozen": ( - initial_logit_delta <= noop_tolerance + actual_lora_parameter_count == expected_lora_parameter_count ), + "no_unexpected_base_trainables": (len(unexpected_base_trainables) == 0), + "frozen_head_gradients_finite": (frozen_head_gradients_finite), + "frozen_encoder_has_no_grads": (frozen_model_grad_count == 0), + "lora_adapter_gradients_finite": (lora_gradients_finite), + "lora_head_gradients_finite": (lora_head_gradients_finite), + "finite_lora_features": bool(torch.isfinite(lora_features_train).all().item()), + "finite_lora_logits": bool(torch.isfinite(lora_logits_train).all().item()), + "initial_logits_match_frozen": (initial_logit_delta <= noop_tolerance), "head_parameter_count_expected": ( head_parameter_count == ( @@ -540,17 +468,10 @@ def main() -> None: "gpu_name": torch.cuda.get_device_name(0), "batch": { "sample_count": len(records), - "annotation_ids": [ - record["annotation_id"] - for record in records - ], - "label_attribute_ids": [ - int(record["attribute_id"]) - for record in records - ], + "annotation_ids": [record["annotation_id"] for record in records], + "label_attribute_ids": [int(record["attribute_id"]) for record in records], "processor_inputs": { - name: list(value.shape) - for name, value in model_inputs.items() + name: list(value.shape) for name, value in model_inputs.items() }, }, "lora_scope": { @@ -565,40 +486,22 @@ def main() -> None: }, "parameters": { "base_model_parameters": int( - sum( - parameter.numel() - for parameter in lora_base_model.parameters() - ) - ), - "classification_head_trainable_parameters": ( - head_parameter_count - ), - "expected_lora_trainable_parameters": ( - expected_lora_parameter_count - ), - "actual_lora_trainable_parameters": ( - actual_lora_parameter_count + sum(parameter.numel() for parameter in lora_base_model.parameters()) ), + "classification_head_trainable_parameters": (head_parameter_count), + "expected_lora_trainable_parameters": (expected_lora_parameter_count), + "actual_lora_trainable_parameters": (actual_lora_parameter_count), "total_lora_plus_head_trainable_parameters": ( - actual_lora_parameter_count - + head_parameter_count - ), - "unexpected_base_trainable_parameter_names": ( - unexpected_base_trainables + actual_lora_parameter_count + head_parameter_count ), + "unexpected_base_trainable_parameter_names": (unexpected_base_trainables), }, "forward_backward": { "frozen_feature_shape": frozen_feature_shape, "frozen_loss": float(frozen_loss.detach().cpu().item()), - "lora_feature_shape": list( - lora_features_train.shape - ), - "lora_loss": float( - lora_loss.detach().cpu().item() - ), - "initial_logit_max_abs_delta": ( - initial_logit_delta - ), + "lora_feature_shape": list(lora_features_train.shape), + "lora_loss": float(lora_loss.detach().cpu().item()), + "initial_logit_max_abs_delta": (initial_logit_delta), "initial_feature_reference_note": ( "Logit equality is tested with the same head " "initialization; default LoRA initialization should " @@ -618,23 +521,14 @@ def main() -> None: print(f"STATUS: {status}") print(f"SELECTED_TARGET_MODULES: {len(selected_targets)}") print(f"INJECTED_TARGET_MODULES: {len(injected_targets)}") - print( - "EXPECTED_LORA_TRAINABLE_PARAMS: " - f"{expected_lora_parameter_count}" - ) - print( - "ACTUAL_LORA_TRAINABLE_PARAMS: " - f"{actual_lora_parameter_count}" - ) + print(f"EXPECTED_LORA_TRAINABLE_PARAMS: {expected_lora_parameter_count}") + print(f"ACTUAL_LORA_TRAINABLE_PARAMS: {actual_lora_parameter_count}") print(f"HEAD_TRAINABLE_PARAMS: {head_parameter_count}") print( "TOTAL_LORA_PLUS_HEAD_TRAINABLE_PARAMS: " f"{actual_lora_parameter_count + head_parameter_count}" ) - print( - "INITIAL_LOGIT_MAX_ABS_DELTA: " - f"{initial_logit_delta:.10f}" - ) + print(f"INITIAL_LOGIT_MAX_ABS_DELTA: {initial_logit_delta:.10f}") print(f"FROZEN_LOSS: {float(frozen_loss.detach().cpu().item()):.6f}") print(f"LORA_LOSS: {float(lora_loss.detach().cpu().item()):.6f}") @@ -650,4 +544,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/rebuild_fashionpedia_neckline_group_stratified_manifest.py b/scripts/rebuild_fashionpedia_neckline_group_stratified_manifest.py index e50804f..e6a7e68 100644 --- a/scripts/rebuild_fashionpedia_neckline_group_stratified_manifest.py +++ b/scripts/rebuild_fashionpedia_neckline_group_stratified_manifest.py @@ -6,7 +6,7 @@ import json import random from collections import Counter, defaultdict -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path @@ -21,7 +21,7 @@ def sha256_file(path: Path) -> str: def stable_seed(base_seed: int, value: str) -> int: - encoded = f"{base_seed}|{value}".encode("utf-8") + encoded = f"{base_seed}|{value}".encode() digest = hashlib.sha256(encoded).hexdigest() return int(digest[:16], 16) @@ -40,13 +40,13 @@ def score_multi_candidate( ( (float(counts[attribute_id]) - target_counts[attribute_id]) / max(target_counts[attribute_id], 1.0) - ) ** 2 + ) + ** 2 for attribute_id in target_counts ) total_error = ( - (float(sum(counts.values())) - total_target) - / max(total_target, 1.0) + (float(sum(counts.values())) - total_target) / max(total_target, 1.0) ) ** 2 overshoot_error = sum( @@ -87,8 +87,7 @@ def main() -> None: if output_dir.exists(): raise FileExistsError( - "Refusing to overwrite manifest output directory: " - f"{output_dir}" + f"Refusing to overwrite manifest output directory: {output_dir}" ) with summary_path.open("r", encoding="utf-8") as handle: @@ -125,20 +124,12 @@ def main() -> None: missing_columns = sorted(required_columns - observed_columns) if missing_columns: - raise ValueError( - "Source manifest columns are missing: " - f"{missing_columns}" - ) + raise ValueError(f"Source manifest columns are missing: {missing_columns}") - annotation_ids = [ - int(record["annotation_id"]) - for record in all_records - ] + annotation_ids = [int(record["annotation_id"]) for record in all_records] if len(annotation_ids) != len(set(annotation_ids)): - raise ValueError( - "Source manifests contain duplicate annotation IDs." - ) + raise ValueError("Source manifests contain duplicate annotation IDs.") records_by_group: dict[int, list[dict[str, str]]] = defaultdict(list) @@ -146,16 +137,10 @@ def main() -> None: group_id = int(record["split_group_id"]) records_by_group[group_id].append(record) - all_attribute_ids = sorted( - { - int(record["attribute_id"]) - for record in all_records - } - ) + all_attribute_ids = sorted({int(record["attribute_id"]) for record in all_records}) attribute_names = { - int(record["attribute_id"]): record["attribute_name"] - for record in all_records + int(record["attribute_id"]): record["attribute_name"] for record in all_records } singleton_groups_by_class: dict[int, list[int]] = defaultdict(list) @@ -164,10 +149,7 @@ def main() -> None: group_buckets: dict[int, str] = {} for group_id, group_records in records_by_group.items(): - class_counts = Counter( - int(record["attribute_id"]) - for record in group_records - ) + class_counts = Counter(int(record["attribute_id"]) for record in group_records) group_class_counts[group_id] = class_counts @@ -192,9 +174,7 @@ def main() -> None: for attribute_id in all_attribute_ids: group_ids = list(singleton_groups_by_class[attribute_id]) - rng = random.Random( - stable_seed(args.seed, f"singleton|{attribute_id}") - ) + rng = random.Random(stable_seed(args.seed, f"singleton|{attribute_id}")) rng.shuffle(group_ids) @@ -208,9 +188,7 @@ def main() -> None: # For multi-instance groups, use deterministic greedy selection. # This preserves multi-instance images in BOTH train and dev while # targeting the class distribution inside this subgroup. - multi_target_group_count = int( - round(len(multi_group_ids) * dev_fraction) - ) + multi_target_group_count = int(round(len(multi_group_ids) * dev_fraction)) multi_target_group_count = max( 1, @@ -227,16 +205,11 @@ def main() -> None: for attribute_id in all_attribute_ids } - multi_target_samples = ( - float(sum(multi_total_counts.values())) * dev_fraction - ) + multi_target_samples = float(sum(multi_total_counts.values())) * dev_fraction rng = random.Random(stable_seed(args.seed, "multi_instance")) - random_tie = { - group_id: rng.random() - for group_id in multi_group_ids - } + random_tie = {group_id: rng.random() for group_id in multi_group_ids} remaining_multi_groups = set(multi_group_ids) selected_multi_groups: set[int] = set() @@ -268,9 +241,7 @@ def main() -> None: best_group_id = group_id if best_group_id is None: - raise RuntimeError( - "Failed to select a multi-instance group for dev." - ) + raise RuntimeError("Failed to select a multi-instance group for dev.") selected_multi_groups.add(best_group_id) selected_multi_counts.update(group_class_counts[best_group_id]) @@ -280,9 +251,7 @@ def main() -> None: train_group_ids = set(records_by_group) - dev_group_ids if not train_group_ids or not dev_group_ids: - raise RuntimeError( - "Rebuild produced an empty train or dev partition." - ) + raise RuntimeError("Rebuild produced an empty train or dev partition.") train_records = [ record @@ -298,32 +267,19 @@ def main() -> None: overlap = train_group_ids & dev_group_ids - total_counts = Counter( - int(record["attribute_id"]) - for record in all_records - ) + total_counts = Counter(int(record["attribute_id"]) for record in all_records) - train_counts = Counter( - int(record["attribute_id"]) - for record in train_records - ) + train_counts = Counter(int(record["attribute_id"]) for record in train_records) - dev_counts = Counter( - int(record["attribute_id"]) - for record in dev_records - ) + dev_counts = Counter(int(record["attribute_id"]) for record in dev_records) def bucket_summary(group_ids: set[int]) -> dict[str, int]: singleton_group_count = sum( - 1 - for group_id in group_ids - if group_buckets[group_id] == "singleton" + 1 for group_id in group_ids if group_buckets[group_id] == "singleton" ) multi_group_count = sum( - 1 - for group_id in group_ids - if group_buckets[group_id] == "multi_instance" + 1 for group_id in group_ids if group_buckets[group_id] == "multi_instance" ) singleton_sample_count = sum( @@ -374,50 +330,34 @@ def bucket_summary(group_ids: set[int]) -> dict[str, int]: class_rows.sort(key=lambda row: row["attribute_id"]) - overall_dev_fraction = ( - float(len(dev_records)) - / float(len(all_records)) - ) + overall_dev_fraction = float(len(dev_records)) / float(len(all_records)) - singleton_dev_group_fraction = ( - float(dev_bucket["singleton_group_count"]) - / float(total_bucket["singleton_group_count"]) + singleton_dev_group_fraction = float(dev_bucket["singleton_group_count"]) / float( + total_bucket["singleton_group_count"] ) - multi_dev_group_fraction = ( - float(dev_bucket["multi_instance_group_count"]) - / float(total_bucket["multi_instance_group_count"]) + multi_dev_group_fraction = float(dev_bucket["multi_instance_group_count"]) / float( + total_bucket["multi_instance_group_count"] ) - singleton_dev_sample_fraction = ( - float(dev_bucket["singleton_sample_count"]) - / float(total_bucket["singleton_sample_count"]) + singleton_dev_sample_fraction = float(dev_bucket["singleton_sample_count"]) / float( + total_bucket["singleton_sample_count"] ) - multi_dev_sample_fraction = ( - float(dev_bucket["multi_instance_sample_count"]) - / float(total_bucket["multi_instance_sample_count"]) - ) + multi_dev_sample_fraction = float( + dev_bucket["multi_instance_sample_count"] + ) / float(total_bucket["multi_instance_sample_count"]) checks = { "source_annotation_coverage_pass": ( len(train_records) + len(dev_records) == len(all_records) ), "group_disjoint_pass": len(overlap) == 0, - "all_classes_in_train_pass": all( - row["train_count"] > 0 - for row in class_rows - ), - "all_classes_in_dev_pass": all( - row["dev_count"] > 0 - for row in class_rows - ), - "overall_dev_fraction_pass": ( - 0.18 <= overall_dev_fraction <= 0.22 - ), + "all_classes_in_train_pass": all(row["train_count"] > 0 for row in class_rows), + "all_classes_in_dev_pass": all(row["dev_count"] > 0 for row in class_rows), + "overall_dev_fraction_pass": (0.18 <= overall_dev_fraction <= 0.22), "class_fraction_pass": all( - 0.17 <= row["dev_fraction"] <= 0.23 - for row in class_rows + 0.17 <= row["dev_fraction"] <= 0.23 for row in class_rows ), "multi_instance_groups_in_train_pass": ( train_bucket["multi_instance_group_count"] > 0 @@ -425,9 +365,7 @@ def bucket_summary(group_ids: set[int]) -> dict[str, int]: "multi_instance_groups_in_dev_pass": ( dev_bucket["multi_instance_group_count"] > 0 ), - "singleton_group_fraction_pass": ( - 0.17 <= singleton_dev_group_fraction <= 0.23 - ), + "singleton_group_fraction_pass": (0.17 <= singleton_dev_group_fraction <= 0.23), "multi_instance_group_fraction_pass": ( 0.17 <= multi_dev_group_fraction <= 0.23 ), @@ -463,7 +401,7 @@ def bucket_summary(group_ids: set[int]) -> dict[str, int]: "schema_version": ( "fashionpedia_neckline_attribute_manifest_v1b_group_stratified" ), - "created_utc": datetime.now(timezone.utc).isoformat(), + "created_utc": datetime.now(UTC).isoformat(), "selection_scope": "train_only", "source_manifest_artifacts": { "source_train_manifest": str(train_path), @@ -483,9 +421,7 @@ def bucket_summary(group_ids: set[int]) -> dict[str, int]: "group_key": "split_group_id=image_id", "dev_fraction": dev_fraction, "seed": int(args.seed), - "singleton_assignment": ( - "deterministic class-stratified group selection" - ), + "singleton_assignment": ("deterministic class-stratified group selection"), "multi_instance_assignment": ( "deterministic greedy group selection with class-distribution " "and group-size preservation" @@ -562,12 +498,12 @@ def bucket_summary(group_ids: set[int]) -> dict[str, int]: for row in class_rows: print( - f'{row["attribute_id"]:>3} ' - f'{row["total_count"]:>5} ' - f'{row["train_count"]:>5} ' - f'{row["dev_count"]:>4} ' - f'{row["dev_fraction"]:>12.3f} ' - f'{row["attribute_name"]}' + f"{row['attribute_id']:>3} " + f"{row['total_count']:>5} " + f"{row['train_count']:>5} " + f"{row['dev_count']:>4} " + f"{row['dev_fraction']:>12.3f} " + f"{row['attribute_name']}" ) print("\n===== MANIFEST CHECKS =====") @@ -583,4 +519,4 @@ def bucket_summary(group_ids: set[int]) -> dict[str, int]: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/run_hog_neckline_baseline.py b/scripts/run_hog_neckline_baseline.py index dff5bb8..04c7c06 100644 --- a/scripts/run_hog_neckline_baseline.py +++ b/scripts/run_hog_neckline_baseline.py @@ -9,14 +9,14 @@ import time import warnings from collections import Counter -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path import joblib import numpy as np import PIL -import sklearn import skimage +import sklearn from PIL import Image from skimage.feature import hog from sklearn.exceptions import ConvergenceWarning @@ -71,9 +71,7 @@ def load_manifest(path: Path) -> list[dict[str, str]]: missing_columns = sorted(required_columns - set(rows[0])) if missing_columns: - raise ValueError( - f"Manifest is missing required columns: {missing_columns}" - ) + raise ValueError(f"Manifest is missing required columns: {missing_columns}") return rows @@ -86,8 +84,7 @@ def crop_and_resize( if not image_path.is_file(): raise FileNotFoundError( - f"Image missing for annotation_id={record['annotation_id']}: " - f"{image_path}" + f"Image missing for annotation_id={record['annotation_id']}: {image_path}" ) try: @@ -149,10 +146,13 @@ def extract_hog_vector( config: dict, ) -> np.ndarray: crop = crop_and_resize(record, config["crop"]) - gray = np.asarray( - crop.convert("L"), - dtype=np.float32, - ) / 255.0 + gray = ( + np.asarray( + crop.convert("L"), + dtype=np.float32, + ) + / 255.0 + ) hog_config = config["hog"] @@ -199,10 +199,7 @@ def build_feature_matrix( features[index] = vector - if ( - index % progress_interval == 0 - or index == len(records) - 1 - ): + if index % progress_interval == 0 or index == len(records) - 1: print( f"feature_progress split={split_name} " f"completed={index + 1}/{len(records)}" @@ -306,8 +303,7 @@ def main() -> None: if manifest_summary.get("status") != "PASS": raise RuntimeError( - "Source manifest is not qualified: " - f"{manifest_summary.get('status')!r}" + f"Source manifest is not qualified: {manifest_summary.get('status')!r}" ) checks = manifest_summary.get("checks", {}) @@ -318,15 +314,9 @@ def main() -> None: train_records = load_manifest(train_manifest_path) dev_records = load_manifest(dev_manifest_path) - train_groups = { - record["split_group_id"] - for record in train_records - } + train_groups = {record["split_group_id"] for record in train_records} - dev_groups = { - record["split_group_id"] - for record in dev_records - } + dev_groups = {record["split_group_id"] for record in dev_records} group_overlap = train_groups & dev_groups @@ -335,25 +325,16 @@ def main() -> None: f"Group leakage detected: {len(group_overlap)} overlapping image groups." ) - target_attribute_ids = [ - int(value) - for value in config["target_attribute_ids"] - ] + target_attribute_ids = [int(value) for value in config["target_attribute_ids"]] target_attribute_set = set(target_attribute_ids) if len(target_attribute_set) != len(target_attribute_ids): raise ValueError("Target attribute IDs are not unique.") - train_attribute_ids = { - int(record["attribute_id"]) - for record in train_records - } + train_attribute_ids = {int(record["attribute_id"]) for record in train_records} - dev_attribute_ids = { - int(record["attribute_id"]) - for record in dev_records - } + dev_attribute_ids = {int(record["attribute_id"]) for record in dev_records} if train_attribute_ids != target_attribute_set: raise ValueError( @@ -373,35 +354,24 @@ def main() -> None: } class_index = { - attribute_id: index - for index, attribute_id in enumerate(target_attribute_ids) + attribute_id: index for index, attribute_id in enumerate(target_attribute_ids) } y_train = np.asarray( - [ - class_index[int(record["attribute_id"])] - for record in train_records - ], + [class_index[int(record["attribute_id"])] for record in train_records], dtype=np.int64, ) y_dev = np.asarray( - [ - class_index[int(record["attribute_id"])] - for record in dev_records - ], + [class_index[int(record["attribute_id"])] for record in dev_records], dtype=np.int64, ) train_class_counts = Counter( - int(record["attribute_id"]) - for record in train_records + int(record["attribute_id"]) for record in train_records ) - dev_class_counts = Counter( - int(record["attribute_id"]) - for record in dev_records - ) + dev_class_counts = Counter(int(record["attribute_id"]) for record in dev_records) print("===== HOG NECKLINE BASELINE PRE-FLIGHT =====") print(f"TRAIN_SAMPLES: {len(train_records)}") @@ -474,24 +444,18 @@ def main() -> None: len(dev_records), len(target_attribute_ids), ): - raise RuntimeError( - "Unexpected probability shape: " - f"{probabilities.shape}" - ) + raise RuntimeError(f"Unexpected probability shape: {probabilities.shape}") if not np.isfinite(probabilities).all(): raise RuntimeError("Probability matrix contains non-finite values.") predicted_indices = probabilities.argmax(axis=1) predicted_attribute_ids = [ - target_attribute_ids[int(index)] - for index in predicted_indices + target_attribute_ids[int(index)] for index in predicted_indices ] accuracy = float(accuracy_score(y_dev, predicted_indices)) - balanced_accuracy = float( - balanced_accuracy_score(y_dev, predicted_indices) - ) + balanced_accuracy = float(balanced_accuracy_score(y_dev, predicted_indices)) macro_f1 = float( f1_score( y_dev, @@ -590,25 +554,18 @@ def main() -> None: "image_path": record["image_path"], "true_attribute_id": int(record["attribute_id"]), "true_attribute_name": record["attribute_name"], - "predicted_attribute_id": int( - predicted_attribute_ids[row_index] - ), + "predicted_attribute_id": int(predicted_attribute_ids[row_index]), "predicted_attribute_name": attribute_names[ predicted_attribute_ids[row_index] ], "confidence": float(probabilities[row_index].max()), "correct": bool( - int(predicted_attribute_ids[row_index]) - == int(record["attribute_id"]) + int(predicted_attribute_ids[row_index]) == int(record["attribute_id"]) ), } - for class_index_value, attribute_id in enumerate( - target_attribute_ids - ): - prediction_row[ - f"probability_attribute_{attribute_id}" - ] = float( + for class_index_value, attribute_id in enumerate(target_attribute_ids): + prediction_row[f"probability_attribute_{attribute_id}"] = float( probabilities[row_index, class_index_value] ) @@ -656,8 +613,7 @@ def main() -> None: "confidence", "correct", ] + [ - f"probability_attribute_{attribute_id}" - for attribute_id in target_attribute_ids + f"probability_attribute_{attribute_id}" for attribute_id in target_attribute_ids ] write_csv( @@ -668,20 +624,16 @@ def main() -> None: confusion_rows = [] - for true_index, true_attribute_id in enumerate( - target_attribute_ids - ): + for true_index, true_attribute_id in enumerate(target_attribute_ids): row = { "true_attribute_id": true_attribute_id, "true_attribute_name": attribute_names[true_attribute_id], } - for predicted_index, predicted_attribute_id in enumerate( - target_attribute_ids - ): - row[ - f"predicted_attribute_{predicted_attribute_id}" - ] = int(confusion[true_index, predicted_index]) + for predicted_index, predicted_attribute_id in enumerate(target_attribute_ids): + row[f"predicted_attribute_{predicted_attribute_id}"] = int( + confusion[true_index, predicted_index] + ) confusion_rows.append(row) @@ -690,7 +642,8 @@ def main() -> None: [ "true_attribute_id", "true_attribute_name", - ] + [ + ] + + [ f"predicted_attribute_{attribute_id}" for attribute_id in target_attribute_ids ], @@ -729,38 +682,22 @@ def main() -> None: classifier = pipeline.named_steps["classifier"] qualification_checks = { - "source_manifest_pass": ( - manifest_summary.get("status") == "PASS" - ), - "source_group_disjoint_pass": bool( - checks.get("group_disjoint_pass", False) - ), + "source_manifest_pass": (manifest_summary.get("status") == "PASS"), + "source_group_disjoint_pass": bool(checks.get("group_disjoint_pass", False)), "rechecked_group_disjoint_pass": len(group_overlap) == 0, - "all_target_classes_in_train": ( - train_attribute_ids == target_attribute_set - ), - "all_target_classes_in_dev": ( - dev_attribute_ids == target_attribute_set - ), - "finite_probability_matrix": bool( - np.isfinite(probabilities).all() - ), - "prediction_count_matches_dev": ( - len(prediction_rows) == len(dev_records) - ), + "all_target_classes_in_train": (train_attribute_ids == target_attribute_set), + "all_target_classes_in_dev": (dev_attribute_ids == target_attribute_set), + "finite_probability_matrix": bool(np.isfinite(probabilities).all()), + "prediction_count_matches_dev": (len(prediction_rows) == len(dev_records)), "no_convergence_warning": convergence_warning_count == 0, } - status = ( - "PASS" - if all(qualification_checks.values()) - else "REVIEW" - ) + status = "PASS" if all(qualification_checks.values()) else "REVIEW" summary = { "status": status, "schema_version": "hog_neckline_baseline_run_v1", - "created_utc": datetime.now(timezone.utc).isoformat(), + "created_utc": datetime.now(UTC).isoformat(), "run_role": config["run_role"], "task_name": config["task_name"], "selection_scope": "train_only_development", @@ -770,9 +707,7 @@ def main() -> None: "manifest_summary": str(manifest_summary_path), "train_manifest_sha256": sha256_file(train_manifest_path), "dev_manifest_sha256": sha256_file(dev_manifest_path), - "manifest_summary_sha256": sha256_file( - manifest_summary_path - ), + "manifest_summary_sha256": sha256_file(manifest_summary_path), "config_sha256": sha256_file(config_path), }, "environment": { @@ -791,15 +726,11 @@ def main() -> None: "group_overlap_count": len(group_overlap), "target_attribute_ids": target_attribute_ids, "train_class_counts": { - str(attribute_id): int( - train_class_counts[attribute_id] - ) + str(attribute_id): int(train_class_counts[attribute_id]) for attribute_id in target_attribute_ids }, "dev_class_counts": { - str(attribute_id): int( - dev_class_counts[attribute_id] - ) + str(attribute_id): int(dev_class_counts[attribute_id]) for attribute_id in target_attribute_ids }, }, @@ -810,9 +741,7 @@ def main() -> None: }, "classifier_specification": { **config["classifier"], - "observed_max_iterations": int( - np.max(classifier.n_iter_) - ), + "observed_max_iterations": int(np.max(classifier.n_iter_)), }, "metrics": { "accuracy": accuracy, @@ -874,12 +803,12 @@ def main() -> None: for row in per_class_rows: print( - f'{row["attribute_id"]:>3} ' - f'{row["support"]:>11} ' - f'{row["precision"]:>9.4f} ' - f'{row["recall"]:>6.4f} ' - f'{row["f1"]:>6.4f} ' - f'{row["attribute_name"]}' + f"{row['attribute_id']:>3} " + f"{row['support']:>11} " + f"{row['precision']:>9.4f} " + f"{row['recall']:>6.4f} " + f"{row['f1']:>6.4f} " + f"{row['attribute_name']}" ) print("") @@ -892,4 +821,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/run_siglip2_frozen_neckline_baseline.py b/scripts/run_siglip2_frozen_neckline_baseline.py index 9265a00..437a0df 100644 --- a/scripts/run_siglip2_frozen_neckline_baseline.py +++ b/scripts/run_siglip2_frozen_neckline_baseline.py @@ -9,7 +9,7 @@ import time import warnings from collections import Counter -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path import joblib @@ -72,9 +72,7 @@ def load_manifest(path: Path) -> list[dict[str, str]]: missing = sorted(required_columns - set(rows[0])) if missing: - raise ValueError( - f"Manifest is missing required columns: {missing}" - ) + raise ValueError(f"Manifest is missing required columns: {missing}") return rows @@ -87,8 +85,7 @@ def crop_from_record( if not image_path.is_file(): raise FileNotFoundError( - "Missing image for annotation_id=" - f"{record['annotation_id']}: {image_path}" + f"Missing image for annotation_id={record['annotation_id']}: {image_path}" ) x = float(record["bbox_x"]) @@ -98,8 +95,7 @@ def crop_from_record( if width <= 0.0 or height <= 0.0: raise ValueError( - "Non-positive bbox for annotation_id=" - f"{record['annotation_id']}" + f"Non-positive bbox for annotation_id={record['annotation_id']}" ) with Image.open(image_path) as source: @@ -122,8 +118,7 @@ def crop_from_record( if right <= left or bottom <= top: raise ValueError( - "Degenerate crop for annotation_id=" - f"{record['annotation_id']}" + f"Degenerate crop for annotation_id={record['annotation_id']}" ) return image.crop((left, top, right, bottom)) @@ -136,10 +131,7 @@ def encode_one_batch( device: torch.device, padding_fraction: float, ) -> np.ndarray: - crops = [ - crop_from_record(record, padding_fraction) - for record in records - ] + crops = [crop_from_record(record, padding_fraction) for record in records] processor_outputs = image_processor( images=crops, @@ -159,22 +151,14 @@ def encode_one_batch( torch.cuda.synchronize(device) - features = ( - image_features.detach() - .float() - .cpu() - .numpy() - ) + features = image_features.detach().float().cpu().numpy() if features.ndim != 2: - raise RuntimeError( - f"Expected rank-2 embeddings, observed {features.shape}" - ) + raise RuntimeError(f"Expected rank-2 embeddings, observed {features.shape}") if features.shape[0] != len(records): raise RuntimeError( - "Embedding batch size mismatch: " - f"{features.shape[0]} vs {len(records)}" + f"Embedding batch size mismatch: {features.shape[0]} vs {len(records)}" ) if not np.isfinite(features).all(): @@ -228,15 +212,10 @@ def extract_embeddings( message = str(exc).lower() is_oom = ( - "out of memory" in message - or "cuda error: out of memory" in message + "out of memory" in message or "cuda error: out of memory" in message ) - if ( - is_oom - and allow_oom_batch_halving - and active_batch_size > 1 - ): + if is_oom and allow_oom_batch_halving and active_batch_size > 1: old_size = active_batch_size active_batch_size = max( 1, @@ -337,15 +316,9 @@ def top_label_ece( upper = edges[index + 1] if index == bin_count - 1: - mask = ( - (confidence >= lower) - & (confidence <= upper) - ) + mask = (confidence >= lower) & (confidence <= upper) else: - mask = ( - (confidence >= lower) - & (confidence < upper) - ) + mask = (confidence >= lower) & (confidence < upper) count = int(mask.sum()) @@ -413,14 +386,10 @@ def main() -> None: config_path, ): if not path.is_file(): - raise FileNotFoundError( - f"Required artifact missing: {path}" - ) + raise FileNotFoundError(f"Required artifact missing: {path}") if run_dir.exists(): - raise FileExistsError( - f"Refusing to overwrite run directory: {run_dir}" - ) + raise FileExistsError(f"Refusing to overwrite run directory: {run_dir}") run_dir.mkdir(parents=True, exist_ok=False) cache_root.mkdir(parents=True, exist_ok=True) @@ -430,8 +399,7 @@ def main() -> None: if manifest_summary.get("status") != "PASS": raise RuntimeError( - "Source manifest summary is not PASS: " - f"{manifest_summary.get('status')!r}" + f"Source manifest summary is not PASS: {manifest_summary.get('status')!r}" ) if not manifest_summary.get( @@ -441,126 +409,75 @@ def main() -> None: "group_disjoint_pass", False, ): - raise RuntimeError( - "Source manifest did not pass group-disjoint validation." - ) + raise RuntimeError("Source manifest did not pass group-disjoint validation.") if not torch.cuda.is_available(): - raise RuntimeError( - "CUDA is unavailable in the active environment." - ) + raise RuntimeError("CUDA is unavailable in the active environment.") device = torch.device("cuda") train_records = load_manifest(train_manifest_path) dev_records = load_manifest(dev_manifest_path) - train_groups = { - record["split_group_id"] - for record in train_records - } + train_groups = {record["split_group_id"] for record in train_records} - dev_groups = { - record["split_group_id"] - for record in dev_records - } + dev_groups = {record["split_group_id"] for record in dev_records} group_overlap = train_groups & dev_groups if group_overlap: raise RuntimeError( - "Group overlap detected between train and dev: " - f"{len(group_overlap)}" + f"Group overlap detected between train and dev: {len(group_overlap)}" ) - target_attribute_ids = [ - int(value) - for value in config["target_attribute_ids"] - ] + target_attribute_ids = [int(value) for value in config["target_attribute_ids"]] target_attribute_set = set(target_attribute_ids) if len(target_attribute_set) != len(target_attribute_ids): - raise RuntimeError( - "Configured target attribute IDs are not unique." - ) + raise RuntimeError("Configured target attribute IDs are not unique.") target_attribute_names = { int(attribute_id): str(name) - for attribute_id, name in config[ - "target_attribute_names" - ].items() + for attribute_id, name in config["target_attribute_names"].items() } - train_attribute_ids = { - int(record["attribute_id"]) - for record in train_records - } + train_attribute_ids = {int(record["attribute_id"]) for record in train_records} - dev_attribute_ids = { - int(record["attribute_id"]) - for record in dev_records - } + dev_attribute_ids = {int(record["attribute_id"]) for record in dev_records} if train_attribute_ids != target_attribute_set: - raise RuntimeError( - "Train label set mismatch: " - f"{sorted(train_attribute_ids)}" - ) + raise RuntimeError(f"Train label set mismatch: {sorted(train_attribute_ids)}") if dev_attribute_ids != target_attribute_set: - raise RuntimeError( - "Dev label set mismatch: " - f"{sorted(dev_attribute_ids)}" - ) + raise RuntimeError(f"Dev label set mismatch: {sorted(dev_attribute_ids)}") class_index = { - attribute_id: index - for index, attribute_id in enumerate( - target_attribute_ids - ) + attribute_id: index for index, attribute_id in enumerate(target_attribute_ids) } y_train = np.asarray( - [ - class_index[int(record["attribute_id"])] - for record in train_records - ], + [class_index[int(record["attribute_id"])] for record in train_records], dtype=np.int64, ) y_dev = np.asarray( - [ - class_index[int(record["attribute_id"])] - for record in dev_records - ], + [class_index[int(record["attribute_id"])] for record in dev_records], dtype=np.int64, ) train_class_counts = Counter( - int(record["attribute_id"]) - for record in train_records + int(record["attribute_id"]) for record in train_records ) - dev_class_counts = Counter( - int(record["attribute_id"]) - for record in dev_records - ) + dev_class_counts = Counter(int(record["attribute_id"]) for record in dev_records) - padding_fraction = float( - config["crop"]["padding_fraction"] - ) + padding_fraction = float(config["crop"]["padding_fraction"]) - initial_batch_size = int( - config["embedding_extraction"][ - "initial_batch_size" - ] - ) + initial_batch_size = int(config["embedding_extraction"]["initial_batch_size"]) allow_oom_batch_halving = bool( - config["embedding_extraction"][ - "allow_oom_batch_halving" - ] + config["embedding_extraction"]["allow_oom_batch_halving"] ) print("===== SIGLIP2 FROZEN NECKLINE PRE-FLIGHT =====") @@ -572,10 +489,7 @@ def main() -> None: print(f"DEVICE: {device}") print(f"MODEL_ID: {config['model']['model_id']}") print(f"INITIAL_BATCH_SIZE: {initial_batch_size}") - print( - "TARGET_ATTRIBUTE_IDS: " - f"{target_attribute_ids}" - ) + print(f"TARGET_ATTRIBUTE_IDS: {target_attribute_ids}") torch.cuda.empty_cache() torch.cuda.reset_peak_memory_stats(device) @@ -640,20 +554,12 @@ def main() -> None: ( "classifier", LogisticRegression( - solver=str( - classifier_config["solver"] - ), + solver=str(classifier_config["solver"]), C=float(classifier_config["C"]), - class_weight=str( - classifier_config["class_weight"] - ), - max_iter=int( - classifier_config["max_iter"] - ), + class_weight=str(classifier_config["class_weight"]), + max_iter=int(classifier_config["max_iter"]), tol=float(classifier_config["tol"]), - random_state=int( - classifier_config["random_state"] - ), + random_state=int(classifier_config["random_state"]), ), ), ], @@ -692,8 +598,7 @@ def main() -> None: expected_classes, ): raise RuntimeError( - "Unexpected classifier class order: " - f"{classifier.classes_.tolist()}" + f"Unexpected classifier class order: {classifier.classes_.tolist()}" ) probabilities = pipeline.predict_proba(dev_features) @@ -704,25 +609,16 @@ def main() -> None: ) if probabilities.shape != expected_probability_shape: - raise RuntimeError( - "Unexpected probability shape: " - f"{probabilities.shape}" - ) + raise RuntimeError(f"Unexpected probability shape: {probabilities.shape}") if not np.isfinite(probabilities).all(): - raise RuntimeError( - "Probability matrix contains non-finite values." - ) + raise RuntimeError("Probability matrix contains non-finite values.") predicted_indices = probabilities.argmax(axis=1) - accuracy = float( - accuracy_score(y_dev, predicted_indices) - ) + accuracy = float(accuracy_score(y_dev, predicted_indices)) - balanced_accuracy = float( - balanced_accuracy_score(y_dev, predicted_indices) - ) + balanced_accuracy = float(balanced_accuracy_score(y_dev, predicted_indices)) macro_f1 = float( f1_score( @@ -742,24 +638,18 @@ def main() -> None: ) ) - precision, recall, f1, support = ( - precision_recall_fscore_support( - y_dev, - predicted_indices, - labels=list( - range(len(target_attribute_ids)) - ), - zero_division=0, - ) + precision, recall, f1, support = precision_recall_fscore_support( + y_dev, + predicted_indices, + labels=list(range(len(target_attribute_ids))), + zero_division=0, ) negative_log_likelihood = float( log_loss( y_dev, probabilities, - labels=list( - range(len(target_attribute_ids)) - ), + labels=list(range(len(target_attribute_ids))), ) ) @@ -800,28 +690,18 @@ def main() -> None: confusion = confusion_matrix( y_dev, predicted_indices, - labels=list( - range(len(target_attribute_ids)) - ), + labels=list(range(len(target_attribute_ids))), ) per_class_rows: list[dict] = [] - for index, attribute_id in enumerate( - target_attribute_ids - ): + for index, attribute_id in enumerate(target_attribute_ids): per_class_rows.append( { "attribute_id": attribute_id, - "attribute_name": target_attribute_names[ - attribute_id - ], - "train_count": int( - train_class_counts[attribute_id] - ), - "dev_count": int( - dev_class_counts[attribute_id] - ), + "attribute_name": target_attribute_names[attribute_id], + "train_count": int(train_class_counts[attribute_id]), + "dev_count": int(dev_class_counts[attribute_id]), "precision": float(precision[index]), "recall": float(recall[index]), "f1": float(f1[index]), @@ -832,44 +712,25 @@ def main() -> None: prediction_rows: list[dict] = [] for index, record in enumerate(dev_records): - predicted_attribute_id = target_attribute_ids[ - int(predicted_indices[index]) - ] + predicted_attribute_id = target_attribute_ids[int(predicted_indices[index])] row = { "annotation_id": record["annotation_id"], "image_id": record["image_id"], "split_group_id": record["split_group_id"], "image_path": record["image_path"], - "true_attribute_id": int( - record["attribute_id"] - ), - "true_attribute_name": record[ - "attribute_name" - ], - "predicted_attribute_id": int( - predicted_attribute_id - ), + "true_attribute_id": int(record["attribute_id"]), + "true_attribute_name": record["attribute_name"], + "predicted_attribute_id": int(predicted_attribute_id), "predicted_attribute_name": ( - target_attribute_names[ - predicted_attribute_id - ] - ), - "confidence": float( - probabilities[index].max() - ), - "correct": bool( - predicted_attribute_id - == int(record["attribute_id"]) + target_attribute_names[predicted_attribute_id] ), + "confidence": float(probabilities[index].max()), + "correct": bool(predicted_attribute_id == int(record["attribute_id"])), } - for class_position, attribute_id in enumerate( - target_attribute_ids - ): - row[ - f"probability_attribute_{attribute_id}" - ] = float( + for class_position, attribute_id in enumerate(target_attribute_ids): + row[f"probability_attribute_{attribute_id}"] = float( probabilities[index, class_position] ) @@ -927,8 +788,7 @@ def main() -> None: "confidence", "correct", ] + [ - f"probability_attribute_{attribute_id}" - for attribute_id in target_attribute_ids + f"probability_attribute_{attribute_id}" for attribute_id in target_attribute_ids ] write_csv( @@ -939,22 +799,14 @@ def main() -> None: confusion_rows: list[dict] = [] - for true_index, true_attribute_id in enumerate( - target_attribute_ids - ): + for true_index, true_attribute_id in enumerate(target_attribute_ids): row = { "true_attribute_id": true_attribute_id, - "true_attribute_name": target_attribute_names[ - true_attribute_id - ], + "true_attribute_name": target_attribute_names[true_attribute_id], } - for predicted_index, predicted_attribute_id in enumerate( - target_attribute_ids - ): - row[ - f"predicted_attribute_{predicted_attribute_id}" - ] = int( + for predicted_index, predicted_attribute_id in enumerate(target_attribute_ids): + row[f"predicted_attribute_{predicted_attribute_id}"] = int( confusion[ true_index, predicted_index, @@ -968,7 +820,8 @@ def main() -> None: [ "true_attribute_id", "true_attribute_name", - ] + [ + ] + + [ f"predicted_attribute_{attribute_id}" for attribute_id in target_attribute_ids ], @@ -979,9 +832,7 @@ def main() -> None: json.dumps( { "top_label_ece": ece, - "bin_count": int( - config["evaluation"]["ece_bins"] - ), + "bin_count": int(config["evaluation"]["ece_bins"]), "bins": ece_bins, }, indent=2, @@ -993,9 +844,7 @@ def main() -> None: json.dumps( { "warning_count": len(warning_rows), - "convergence_warning_count": ( - convergence_warning_count - ), + "convergence_warning_count": (convergence_warning_count), "warnings": warning_rows, }, indent=2, @@ -1006,15 +855,10 @@ def main() -> None: extraction = { "train": train_extract_info, "dev": dev_extract_info, - "embedding_dimension": int( - train_features.shape[1] - ), - "embedding_normalization": ( - config["model"]["embedding_normalization"] - ), + "embedding_dimension": int(train_features.shape[1]), + "embedding_normalization": (config["model"]["embedding_normalization"]), "gpu_peak_memory_allocated_mib": round( - torch.cuda.max_memory_allocated(device) - / (1024 ** 2), + torch.cuda.max_memory_allocated(device) / (1024**2), 2, ), } @@ -1030,9 +874,7 @@ def main() -> None: ) qualification_checks = { - "source_manifest_pass": ( - manifest_summary.get("status") == "PASS" - ), + "source_manifest_pass": (manifest_summary.get("status") == "PASS"), "source_group_disjoint_pass": bool( manifest_summary.get( "checks", @@ -1042,79 +884,39 @@ def main() -> None: False, ) ), - "rechecked_group_disjoint_pass": ( - len(group_overlap) == 0 - ), - "cuda_available": bool( - torch.cuda.is_available() - ), - "all_target_classes_in_train": ( - train_attribute_ids - == target_attribute_set - ), - "all_target_classes_in_dev": ( - dev_attribute_ids - == target_attribute_set - ), + "rechecked_group_disjoint_pass": (len(group_overlap) == 0), + "cuda_available": bool(torch.cuda.is_available()), + "all_target_classes_in_train": (train_attribute_ids == target_attribute_set), + "all_target_classes_in_dev": (dev_attribute_ids == target_attribute_set), "train_embedding_count_matches": ( - train_features.shape[0] - == len(train_records) - ), - "dev_embedding_count_matches": ( - dev_features.shape[0] - == len(dev_records) + train_features.shape[0] == len(train_records) ), + "dev_embedding_count_matches": (dev_features.shape[0] == len(dev_records)), "matching_embedding_dimension": ( - train_features.shape[1] - == dev_features.shape[1] - ), - "finite_probability_matrix": bool( - np.isfinite(probabilities).all() - ), - "prediction_count_matches_dev": ( - len(prediction_rows) - == len(dev_records) - ), - "no_convergence_warning": ( - convergence_warning_count == 0 + train_features.shape[1] == dev_features.shape[1] ), + "finite_probability_matrix": bool(np.isfinite(probabilities).all()), + "prediction_count_matches_dev": (len(prediction_rows) == len(dev_records)), + "no_convergence_warning": (convergence_warning_count == 0), } - status = ( - "PASS" - if all(qualification_checks.values()) - else "REVIEW" - ) + status = "PASS" if all(qualification_checks.values()) else "REVIEW" summary = { "status": status, - "schema_version": ( - "siglip2_frozen_neckline_baseline_run_v1" - ), - "created_utc": datetime.now( - timezone.utc - ).isoformat(), + "schema_version": ("siglip2_frozen_neckline_baseline_run_v1"), + "created_utc": datetime.now(UTC).isoformat(), "run_role": config["run_role"], "task_name": config["task_name"], "selection_scope": "train_only_development", "source_artifacts": { "train_manifest": str(train_manifest_path), "dev_manifest": str(dev_manifest_path), - "manifest_summary": str( - manifest_summary_path - ), - "train_manifest_sha256": sha256_file( - train_manifest_path - ), - "dev_manifest_sha256": sha256_file( - dev_manifest_path - ), - "manifest_summary_sha256": sha256_file( - manifest_summary_path - ), - "config_sha256": sha256_file( - config_path - ), + "manifest_summary": str(manifest_summary_path), + "train_manifest_sha256": sha256_file(train_manifest_path), + "dev_manifest_sha256": sha256_file(dev_manifest_path), + "manifest_summary_sha256": sha256_file(manifest_summary_path), + "config_sha256": sha256_file(config_path), }, "environment": { "python_executable": sys.executable, @@ -1131,19 +933,14 @@ def main() -> None: "model": { "model_id": config["model"]["model_id"], "model_class": model.__class__.__name__, - "image_processor_class": ( - image_processor.__class__.__name__ - ), + "image_processor_class": (image_processor.__class__.__name__), "model_commit_hash": getattr( model.config, "_commit_hash", None, ), "model_parameter_count": int( - sum( - parameter.numel() - for parameter in model.parameters() - ) + sum(parameter.numel() for parameter in model.parameters()) ), "encoder_trainable": False, }, @@ -1155,33 +952,25 @@ def main() -> None: "group_overlap_count": len(group_overlap), "target_attribute_ids": target_attribute_ids, "train_class_counts": { - str(attribute_id): int( - train_class_counts[attribute_id] - ) + str(attribute_id): int(train_class_counts[attribute_id]) for attribute_id in target_attribute_ids }, "dev_class_counts": { - str(attribute_id): int( - dev_class_counts[attribute_id] - ) + str(attribute_id): int(dev_class_counts[attribute_id]) for attribute_id in target_attribute_ids }, }, "embedding_extraction": extraction, "classifier_specification": { **config["classifier"], - "observed_max_iterations": int( - np.max(classifier.n_iter_) - ), + "observed_max_iterations": int(np.max(classifier.n_iter_)), }, "metrics": { "accuracy": accuracy, "balanced_accuracy": balanced_accuracy, "macro_f1": macro_f1, "weighted_f1": weighted_f1, - "negative_log_likelihood": ( - negative_log_likelihood - ), + "negative_log_likelihood": (negative_log_likelihood), "multiclass_brier": multiclass_brier, "top_label_ece": ece, "top_2_accuracy": top_2_accuracy, @@ -1190,36 +979,20 @@ def main() -> None: "qualification_checks": qualification_checks, "warnings": { "warning_count": len(warning_rows), - "convergence_warning_count": ( - convergence_warning_count - ), + "convergence_warning_count": (convergence_warning_count), }, "artifacts": { - "train_embeddings_l2_npy": str( - train_embeddings_path - ), - "dev_embeddings_l2_npy": str( - dev_embeddings_path - ), + "train_embeddings_l2_npy": str(train_embeddings_path), + "dev_embeddings_l2_npy": str(dev_embeddings_path), "train_labels_npy": str(train_labels_path), "dev_labels_npy": str(dev_labels_path), "model_joblib": str(model_path), - "per_class_metrics_csv": str( - per_class_path - ), - "dev_predictions_csv": str( - predictions_path - ), - "confusion_matrix_csv": str( - confusion_path - ), + "per_class_metrics_csv": str(per_class_path), + "dev_predictions_csv": str(predictions_path), + "confusion_matrix_csv": str(confusion_path), "ece_bins_json": str(ece_path), - "training_warnings_json": str( - warnings_path - ), - "embedding_extraction_json": str( - extraction_path - ), + "training_warnings_json": str(warnings_path), + "embedding_extraction_json": str(extraction_path), "config_copy_json": str(config_copy_path), }, "elapsed_seconds": round( @@ -1240,29 +1013,16 @@ def main() -> None: print(f"TRAIN_SAMPLES: {len(train_records)}") print(f"DEV_SAMPLES: {len(dev_records)}") print(f"GROUP_OVERLAP_COUNT: {len(group_overlap)}") - print( - "EMBEDDING_DIMENSION: " - f"{train_features.shape[1]}" - ) + print(f"EMBEDDING_DIMENSION: {train_features.shape[1]}") print(f"ACCURACY: {accuracy:.6f}") - print( - f"BALANCED_ACCURACY: {balanced_accuracy:.6f}" - ) + print(f"BALANCED_ACCURACY: {balanced_accuracy:.6f}") print(f"MACRO_F1: {macro_f1:.6f}") print(f"WEIGHTED_F1: {weighted_f1:.6f}") - print( - "NEGATIVE_LOG_LIKELIHOOD: " - f"{negative_log_likelihood:.6f}" - ) - print( - f"MULTICLASS_BRIER: {multiclass_brier:.6f}" - ) + print(f"NEGATIVE_LOG_LIKELIHOOD: {negative_log_likelihood:.6f}") + print(f"MULTICLASS_BRIER: {multiclass_brier:.6f}") print(f"TOP_LABEL_ECE: {ece:.6f}") print(f"TOP_2_ACCURACY: {top_2_accuracy:.6f}") - print( - "CONVERGENCE_WARNING_COUNT: " - f"{convergence_warning_count}" - ) + print(f"CONVERGENCE_WARNING_COUNT: {convergence_warning_count}") print( "GPU_PEAK_MEMORY_ALLOCATED_MIB: " f"{extraction['gpu_peak_memory_allocated_mib']:.2f}" @@ -1270,18 +1030,16 @@ def main() -> None: print("") print("===== PER-CLASS METRICS =====") - print( - "ID DEV_SUPPORT PRECISION RECALL F1 ATTRIBUTE" - ) + print("ID DEV_SUPPORT PRECISION RECALL F1 ATTRIBUTE") for row in per_class_rows: print( - f'{row["attribute_id"]:>3} ' - f'{row["support"]:>11} ' - f'{row["precision"]:>9.4f} ' - f'{row["recall"]:>6.4f} ' - f'{row["f1"]:>6.4f} ' - f'{row["attribute_name"]}' + f"{row['attribute_id']:>3} " + f"{row['support']:>11} " + f"{row['precision']:>9.4f} " + f"{row['recall']:>6.4f} " + f"{row['f1']:>6.4f} " + f"{row['attribute_name']}" ) print("") @@ -1295,4 +1053,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/run_siglip2_lora_matched_head_development.py b/scripts/run_siglip2_lora_matched_head_development.py index 2b66ca7..d4be705 100644 --- a/scripts/run_siglip2_lora_matched_head_development.py +++ b/scripts/run_siglip2_lora_matched_head_development.py @@ -12,7 +12,7 @@ import sys import time from collections import Counter -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path import joblib @@ -22,8 +22,8 @@ import torch import torch.nn.functional as F import transformers -from PIL import Image from peft import LoraConfig, get_peft_model +from PIL import Image from sklearn.metrics import ( accuracy_score, balanced_accuracy_score, @@ -34,7 +34,6 @@ ) from transformers import AutoImageProcessor, AutoModel - ENCODER_SELF_ATTN_PATTERN = re.compile( r"^vision_model\.encoder\.layers\.\d+\.self_attn\." r"(q_proj|k_proj|v_proj|out_proj)$" @@ -97,9 +96,7 @@ def load_manifest(path: Path) -> list[dict[str, str]]: missing = sorted(required_columns - set(rows[0])) if missing: - raise ValueError( - f"Manifest missing required columns: {missing}" - ) + raise ValueError(f"Manifest missing required columns: {missing}") return rows @@ -120,8 +117,7 @@ def crop_from_record( if not image_path.is_file(): raise FileNotFoundError( - "Missing image for annotation_id=" - f"{record['annotation_id']}: {image_path}" + f"Missing image for annotation_id={record['annotation_id']}: {image_path}" ) x = float(record["bbox_x"]) @@ -131,8 +127,7 @@ def crop_from_record( if width <= 0.0 or height <= 0.0: raise ValueError( - "Non-positive bbox for annotation_id=" - f"{record['annotation_id']}" + f"Non-positive bbox for annotation_id={record['annotation_id']}" ) with Image.open(image_path) as source: @@ -155,8 +150,7 @@ def crop_from_record( if right <= left or bottom <= top: raise ValueError( - "Degenerate crop for annotation_id=" - f"{record['annotation_id']}" + f"Degenerate crop for annotation_id={record['annotation_id']}" ) return image.crop((left, top, right, bottom)) @@ -168,10 +162,7 @@ def make_model_inputs( device: torch.device, padding_fraction: float, ) -> dict[str, torch.Tensor]: - crops = [ - crop_from_record(record, padding_fraction) - for record in records - ] + crops = [crop_from_record(record, padding_fraction) for record in records] processor_outputs = image_processor( images=crops, @@ -203,15 +194,9 @@ def top_label_ece( upper = float(edges[index + 1]) if index == bin_count - 1: - mask = ( - (confidence >= lower) - & (confidence <= upper) - ) + mask = (confidence >= lower) & (confidence <= upper) else: - mask = ( - (confidence >= lower) - & (confidence < upper) - ) + mask = (confidence >= lower) & (confidence < upper) count = int(mask.sum()) @@ -222,11 +207,7 @@ def top_label_ece( bin_accuracy = float(correct[mask].mean()) absolute_gap = abs(mean_confidence - bin_accuracy) - ece += ( - float(count) - / float(len(targets)) - * absolute_gap - ) + ece += float(count) / float(len(targets)) * absolute_gap rows.append( { @@ -243,12 +224,7 @@ def top_label_ece( def count_parameters(module: torch.nn.Module) -> int: - return int( - sum( - parameter.numel() - for parameter in module.parameters() - ) - ) + return int(sum(parameter.numel() for parameter in module.parameters())) def count_trainable_parameters(module: torch.nn.Module) -> int: @@ -280,17 +256,14 @@ def all_trainable_gradients_finite( for module in modules: trainable_parameters.extend( - parameter - for parameter in module.parameters() - if parameter.requires_grad + parameter for parameter in module.parameters() if parameter.requires_grad ) if not trainable_parameters: return False return all( - parameter.grad is not None - and torch.isfinite(parameter.grad).all().item() + parameter.grad is not None and torch.isfinite(parameter.grad).all().item() for parameter in trainable_parameters ) @@ -303,15 +276,10 @@ def build_class_weights( counts = np.bincount(labels, minlength=class_count) if np.any(counts <= 0): - raise RuntimeError( - "Every target class must appear in training labels." - ) + raise RuntimeError("Every target class must appear in training labels.") total = float(labels.shape[0]) - raw_weights = total / ( - float(class_count) - * counts.astype(np.float64) - ) + raw_weights = total / (float(class_count) * counts.astype(np.float64)) weights = torch.tensor( raw_weights, @@ -320,8 +288,7 @@ def build_class_weights( ) return weights, { - str(index): float(raw_weights[index]) - for index in range(class_count) + str(index): float(raw_weights[index]) for index in range(class_count) } @@ -380,17 +347,12 @@ def evaluate( weight=class_weights, ) - logits_chunks.append( - logits.detach().float().cpu().numpy() - ) + logits_chunks.append(logits.detach().float().cpu().numpy()) - target_chunks.append( - batch_targets.detach().cpu().numpy() - ) + target_chunks.append(batch_targets.detach().cpu().numpy()) - weighted_loss_sum += ( - float(loss.detach().float().cpu().item()) - * len(batch_records) + weighted_loss_sum += float(loss.detach().float().cpu().item()) * len( + batch_records ) sample_count += len(batch_records) @@ -402,15 +364,10 @@ def evaluate( len(records), len(target_attribute_ids), ): - raise RuntimeError( - "Unexpected evaluation logit shape: " - f"{logits_all.shape}" - ) + raise RuntimeError(f"Unexpected evaluation logit shape: {logits_all.shape}") if not np.isfinite(logits_all).all(): - raise RuntimeError( - "Non-finite logits encountered during evaluation." - ) + raise RuntimeError("Non-finite logits encountered during evaluation.") shifted = logits_all - logits_all.max( axis=1, @@ -424,19 +381,13 @@ def evaluate( ) if not np.isfinite(probabilities).all(): - raise RuntimeError( - "Non-finite probabilities encountered." - ) + raise RuntimeError("Non-finite probabilities encountered.") predicted_indices = probabilities.argmax(axis=1) - accuracy = float( - accuracy_score(y_true, predicted_indices) - ) + accuracy = float(accuracy_score(y_true, predicted_indices)) - balanced_accuracy = float( - balanced_accuracy_score(y_true, predicted_indices) - ) + balanced_accuracy = float(balanced_accuracy_score(y_true, predicted_indices)) macro_f1 = float( f1_score( @@ -498,13 +449,11 @@ def evaluate( ) ) - precision, recall, f1, support = ( - precision_recall_fscore_support( - y_true, - predicted_indices, - labels=list(range(len(target_attribute_ids))), - zero_division=0, - ) + precision, recall, f1, support = precision_recall_fscore_support( + y_true, + predicted_indices, + labels=list(range(len(target_attribute_ids))), + zero_division=0, ) confusion = confusion_matrix( @@ -515,15 +464,11 @@ def evaluate( per_class_rows: list[dict] = [] - for class_index, attribute_id in enumerate( - target_attribute_ids - ): + for class_index, attribute_id in enumerate(target_attribute_ids): per_class_rows.append( { "attribute_id": int(attribute_id), - "attribute_name": target_attribute_names[ - int(attribute_id) - ], + "attribute_name": target_attribute_names[int(attribute_id)], "precision": float(precision[class_index]), "recall": float(recall[class_index]), "f1": float(f1[class_index]), @@ -534,44 +479,25 @@ def evaluate( prediction_rows: list[dict] = [] for row_index, record in enumerate(records): - predicted_attribute_id = target_attribute_ids[ - int(predicted_indices[row_index]) - ] + predicted_attribute_id = target_attribute_ids[int(predicted_indices[row_index])] row = { "annotation_id": record["annotation_id"], "image_id": record["image_id"], "split_group_id": record["split_group_id"], "image_path": record["image_path"], - "true_attribute_id": int( - record["attribute_id"] - ), - "true_attribute_name": record[ - "attribute_name" - ], - "predicted_attribute_id": int( - predicted_attribute_id - ), + "true_attribute_id": int(record["attribute_id"]), + "true_attribute_name": record["attribute_name"], + "predicted_attribute_id": int(predicted_attribute_id), "predicted_attribute_name": ( - target_attribute_names[ - predicted_attribute_id - ] - ), - "confidence": float( - probabilities[row_index].max() - ), - "correct": bool( - predicted_attribute_id - == int(record["attribute_id"]) + target_attribute_names[predicted_attribute_id] ), + "confidence": float(probabilities[row_index].max()), + "correct": bool(predicted_attribute_id == int(record["attribute_id"])), } - for class_position, attribute_id in enumerate( - target_attribute_ids - ): - row[ - f"probability_attribute_{attribute_id}" - ] = float( + for class_position, attribute_id in enumerate(target_attribute_ids): + row[f"probability_attribute_{attribute_id}"] = float( probabilities[row_index, class_position] ) @@ -579,31 +505,21 @@ def evaluate( confusion_rows: list[dict] = [] - for true_index, true_attribute_id in enumerate( - target_attribute_ids - ): + for true_index, true_attribute_id in enumerate(target_attribute_ids): row = { "true_attribute_id": int(true_attribute_id), - "true_attribute_name": target_attribute_names[ - int(true_attribute_id) - ], + "true_attribute_name": target_attribute_names[int(true_attribute_id)], } - for predicted_index, predicted_attribute_id in enumerate( - target_attribute_ids - ): - row[ - f"predicted_attribute_{predicted_attribute_id}" - ] = int( + for predicted_index, predicted_attribute_id in enumerate(target_attribute_ids): + row[f"predicted_attribute_{predicted_attribute_id}"] = int( confusion[true_index, predicted_index] ) confusion_rows.append(row) metrics = { - "evaluation_weighted_cross_entropy": ( - weighted_loss_sum / float(sample_count) - ), + "evaluation_weighted_cross_entropy": (weighted_loss_sum / float(sample_count)), "accuracy": accuracy, "balanced_accuracy": balanced_accuracy, "macro_f1": macro_f1, @@ -614,10 +530,15 @@ def evaluate( "top_2_accuracy": top_2_accuracy, } - return metrics, per_class_rows, prediction_rows, { - "ece_rows": ece_rows, - "confusion_rows": confusion_rows, - } + return ( + metrics, + per_class_rows, + prediction_rows, + { + "ece_rows": ece_rows, + "confusion_rows": confusion_rows, + }, + ) def save_best_checkpoint( @@ -657,11 +578,7 @@ def save_best_checkpoint( "arm_name": arm_name, "epoch": int(epoch), "head_state_path": str(head_path), - "adapter_directory": ( - None - if adapter_dir is None - else str(adapter_dir) - ), + "adapter_directory": (None if adapter_dir is None else str(adapter_dir)), "selection_metrics": metrics, } @@ -669,11 +586,7 @@ def save_best_checkpoint( return { "head_state_path": str(head_path), - "adapter_directory": ( - None - if adapter_dir is None - else str(adapter_dir) - ), + "adapter_directory": (None if adapter_dir is None else str(adapter_dir)), "metadata_path": str(metadata_path), } @@ -699,9 +612,7 @@ def build_arm( int(config["head"]["output_classes"]), ) - head.load_state_dict( - copy.deepcopy(initial_head_state) - ) + head.load_state_dict(copy.deepcopy(initial_head_state)) head.to(device) @@ -747,17 +658,13 @@ def build_arm( ] adapter_parameter_count = int( - sum( - parameter.numel() - for _, parameter in adapter_trainable - ) + sum(parameter.numel() for _, parameter in adapter_trainable) ) unexpected_base_trainables = [ name for name, _ in adapter_trainable - if ".lora_A." not in name - and ".lora_B." not in name + if ".lora_A." not in name and ".lora_B." not in name ] return { @@ -766,13 +673,8 @@ def build_arm( "mode_module": lora_model, "head": head, "adapter_parameter_count": adapter_parameter_count, - "adapter_trainable_names": [ - name - for name, _ in adapter_trainable - ], - "unexpected_base_trainables": ( - unexpected_base_trainables - ), + "adapter_trainable_names": [name for name, _ in adapter_trainable], + "unexpected_base_trainables": (unexpected_base_trainables), "selected_target_modules": target_modules, } @@ -806,9 +708,7 @@ def train_one_epoch( head.train() optimizer.zero_grad(set_to_none=True) - order = np.random.default_rng( - seed + epoch - ).permutation(len(train_records)) + order = np.random.default_rng(seed + epoch).permutation(len(train_records)) total_loss = 0.0 total_samples = 0 @@ -828,10 +728,7 @@ def train_one_epoch( ) batch_indices = order[start:end] - batch_records = [ - train_records[int(index)] - for index in batch_indices - ] + batch_records = [train_records[int(index)] for index in batch_indices] batch_targets = torch.tensor( train_labels[batch_indices], @@ -883,26 +780,17 @@ def train_one_epoch( weight=class_weights, ) - scaled_loss = ( - raw_loss - / float(gradient_accumulation_steps) - ) + scaled_loss = raw_loss / float(gradient_accumulation_steps) scaler.scale(scaled_loss).backward() - total_loss += ( - float(raw_loss.detach().float().cpu().item()) - * len(batch_records) - ) + total_loss += float(raw_loss.detach().float().cpu().item()) * len(batch_records) total_samples += len(batch_records) batch_counter += 1 is_final_batch = end == len(train_records) - should_step = ( - batch_counter % gradient_accumulation_steps == 0 - or is_final_batch - ) + should_step = batch_counter % gradient_accumulation_steps == 0 or is_final_batch if should_step: scaler.unscale_(optimizer) @@ -912,9 +800,7 @@ def train_one_epoch( if arm_name == "lora": trainable_modules.append(mode_module) - gradients_finite = all_trainable_gradients_finite( - trainable_modules - ) + gradients_finite = all_trainable_gradients_finite(trainable_modules) if not gradients_finite: nonfinite_gradient_detected = True @@ -939,13 +825,9 @@ def train_one_epoch( optimizer_steps += 1 return { - "train_weighted_cross_entropy": ( - total_loss / float(total_samples) - ), + "train_weighted_cross_entropy": (total_loss / float(total_samples)), "optimizer_steps": optimizer_steps, - "nonfinite_gradient_detected": ( - nonfinite_gradient_detected - ), + "nonfinite_gradient_detected": (nonfinite_gradient_detected), } @@ -989,12 +871,8 @@ def run_arm( [ { "params": list(head.parameters()), - "lr": float( - config["training"]["head_learning_rate"] - ), - "weight_decay": float( - config["training"]["weight_decay"] - ), + "lr": float(config["training"]["head_learning_rate"]), + "weight_decay": float(config["training"]["weight_decay"]), } ] ) @@ -1009,21 +887,13 @@ def run_arm( [ { "params": adapter_parameters, - "lr": float( - config["training"]["lora_learning_rate"] - ), - "weight_decay": float( - config["training"]["weight_decay"] - ), + "lr": float(config["training"]["lora_learning_rate"]), + "weight_decay": float(config["training"]["weight_decay"]), }, { "params": list(head.parameters()), - "lr": float( - config["training"]["head_learning_rate"] - ), - "weight_decay": float( - config["training"]["weight_decay"] - ), + "lr": float(config["training"]["head_learning_rate"]), + "weight_decay": float(config["training"]["weight_decay"]), }, ] ) @@ -1039,19 +909,11 @@ def run_arm( ) epochs = int(config["training"]["epochs"]) - batch_size = int( - config["training"]["physical_batch_size"] - ) - gradient_accumulation_steps = int( - config["training"]["gradient_accumulation_steps"] - ) + batch_size = int(config["training"]["physical_batch_size"]) + gradient_accumulation_steps = int(config["training"]["gradient_accumulation_steps"]) seed = int(config["training"]["seed"]) - padding_fraction = float( - config["crop"]["padding_fraction"] - ) - max_grad_norm = float( - config["training"]["max_grad_norm"] - ) + padding_fraction = float(config["crop"]["padding_fraction"]) + max_grad_norm = float(config["training"]["max_grad_norm"]) ece_bins = int(config["evaluation"]["ece_bins"]) epoch_rows: list[dict] = [] @@ -1108,9 +970,7 @@ def run_arm( ece_bins=ece_bins, ) - epoch_elapsed = ( - time.perf_counter() - epoch_start - ) + epoch_elapsed = time.perf_counter() - epoch_start epoch_row = { "epoch": epoch, @@ -1137,17 +997,10 @@ def run_arm( if best_metrics is None: is_better = True - elif ( - dev_metrics["macro_f1"] - > best_metrics["macro_f1"] + 1e-12 - ): + elif dev_metrics["macro_f1"] > best_metrics["macro_f1"] + 1e-12: is_better = True elif ( - abs( - dev_metrics["macro_f1"] - - best_metrics["macro_f1"] - ) - <= 1e-12 + abs(dev_metrics["macro_f1"] - best_metrics["macro_f1"]) <= 1e-12 and dev_metrics["negative_log_likelihood"] < best_metrics["negative_log_likelihood"] ): @@ -1177,9 +1030,7 @@ def run_arm( ) if best_epoch is None: - raise RuntimeError( - f"No best epoch selected for arm={arm_name}" - ) + raise RuntimeError(f"No best epoch selected for arm={arm_name}") epoch_metrics_path = arm_dir / "epoch_metrics.csv" per_class_path = arm_dir / "best_dev_per_class_metrics.csv" @@ -1219,8 +1070,7 @@ def run_arm( "confidence", "correct", ] + [ - f"probability_attribute_{attribute_id}" - for attribute_id in target_attribute_ids + f"probability_attribute_{attribute_id}" for attribute_id in target_attribute_ids ] write_csv( @@ -1234,7 +1084,8 @@ def run_arm( fieldnames=[ "true_attribute_id", "true_attribute_name", - ] + [ + ] + + [ f"predicted_attribute_{attribute_id}" for attribute_id in target_attribute_ids ], @@ -1244,43 +1095,30 @@ def run_arm( write_json( ece_path, { - "top_label_ece": ( - best_metrics["top_label_ece"] - ), - "bin_count": int( - config["evaluation"]["ece_bins"] - ), + "top_label_ece": (best_metrics["top_label_ece"]), + "bin_count": int(config["evaluation"]["ece_bins"]), "bins": best_extra["ece_rows"], }, ) - expected_head_parameters = ( - int(config["head"]["input_dimension"]) - * int(config["head"]["output_classes"]) - + int(config["head"]["output_classes"]) - ) + expected_head_parameters = int(config["head"]["input_dimension"]) * int( + config["head"]["output_classes"] + ) + int(config["head"]["output_classes"]) checks = { "best_epoch_selected": best_epoch is not None, - "finite_selected_macro_f1": bool( - np.isfinite(best_metrics["macro_f1"]) - ), + "finite_selected_macro_f1": bool(np.isfinite(best_metrics["macro_f1"])), "finite_selected_nll": bool( - np.isfinite( - best_metrics["negative_log_likelihood"] - ) + np.isfinite(best_metrics["negative_log_likelihood"]) ), "all_epoch_optimizer_steps_positive": all( - row["optimizer_steps"] > 0 - for row in epoch_rows + row["optimizer_steps"] > 0 for row in epoch_rows ), "no_nonfinite_gradients": not any( - row["nonfinite_gradient_detected"] - for row in epoch_rows + row["nonfinite_gradient_detected"] for row in epoch_rows ), "head_parameter_count_expected": ( - count_parameters(head) - == expected_head_parameters + count_parameters(head) == expected_head_parameters ), "best_prediction_count_matches_dev": ( len(best_predictions) == len(dev_records) @@ -1292,27 +1130,17 @@ def run_arm( arm["adapter_parameter_count"] == 0 ) else: - checks["lora_adapter_parameter_count_matches"] = ( - arm["adapter_parameter_count"] - == int( - config["lora"][ - "expected_adapter_parameter_count" - ] - ) - ) + checks["lora_adapter_parameter_count_matches"] = arm[ + "adapter_parameter_count" + ] == int(config["lora"]["expected_adapter_parameter_count"]) checks["no_unexpected_base_trainables"] = ( len(arm["unexpected_base_trainables"]) == 0 ) - checks["lora_target_count_matches"] = ( - len(arm["selected_target_modules"]) - == int( - config["lora"][ - "expected_target_module_count" - ] - ) - ) + checks["lora_target_count_matches"] = len( + arm["selected_target_modules"] + ) == int(config["lora"]["expected_target_module_count"]) status = "PASS" if all(checks.values()) else "REVIEW" @@ -1327,26 +1155,19 @@ def run_arm( "classification_head": count_parameters(head), "lora_adapter": arm["adapter_parameter_count"], "total_trainable": ( - count_parameters(head) - + arm["adapter_parameter_count"] + count_parameters(head) + arm["adapter_parameter_count"] ), }, "lora_scope": { - "selected_target_modules": ( - arm["selected_target_modules"] - ), - "unexpected_base_trainables": ( - arm["unexpected_base_trainables"] - ), + "selected_target_modules": (arm["selected_target_modules"]), + "unexpected_base_trainables": (arm["unexpected_base_trainables"]), }, "qualification_checks": checks, "artifacts": { "epoch_metrics_csv": str(epoch_metrics_path), "best_per_class_metrics_csv": str(per_class_path), "best_dev_predictions_csv": str(predictions_path), - "best_dev_confusion_matrix_csv": str( - confusion_path - ), + "best_dev_confusion_matrix_csv": str(confusion_path), "best_dev_ece_bins_json": str(ece_path), **best_artifacts, }, @@ -1397,14 +1218,10 @@ def main() -> None: train_manifest_path = Path(args.train_manifest).resolve() dev_manifest_path = Path(args.dev_manifest).resolve() - manifest_summary_path = Path( - args.manifest_summary - ).resolve() + manifest_summary_path = Path(args.manifest_summary).resolve() hog_config_path = Path(args.hog_config).resolve() lora_audit_path = Path(args.lora_audit_json).resolve() - lora_preflight_path = Path( - args.lora_preflight_json - ).resolve() + lora_preflight_path = Path(args.lora_preflight_json).resolve() config_path = Path(args.config).resolve() cache_root = Path(args.cache_dir).resolve() run_dir = Path(args.run_dir).resolve() @@ -1421,19 +1238,13 @@ def main() -> None: for path in required_paths: if not path.is_file(): - raise FileNotFoundError( - f"Required artifact missing: {path}" - ) + raise FileNotFoundError(f"Required artifact missing: {path}") if run_dir.exists(): - raise FileExistsError( - f"Refusing to overwrite run directory: {run_dir}" - ) + raise FileExistsError(f"Refusing to overwrite run directory: {run_dir}") if not torch.cuda.is_available(): - raise RuntimeError( - "CUDA is unavailable in the active environment." - ) + raise RuntimeError("CUDA is unavailable in the active environment.") device = torch.device("cuda") @@ -1449,9 +1260,7 @@ def main() -> None: config = load_json(config_path) if manifest_summary.get("status") != "PASS": - raise RuntimeError( - "Source manifest summary is not PASS." - ) + raise RuntimeError("Source manifest summary is not PASS.") if not manifest_summary.get( "checks", @@ -1460,102 +1269,59 @@ def main() -> None: "group_disjoint_pass", False, ): - raise RuntimeError( - "Source manifest failed group-disjoint check." - ) + raise RuntimeError("Source manifest failed group-disjoint check.") if lora_audit.get("status") != "PASS": - raise RuntimeError( - "LoRA target audit is not PASS." - ) + raise RuntimeError("LoRA target audit is not PASS.") if lora_preflight.get("status") != "PASS": - raise RuntimeError( - "LoRA matched-head preflight is not PASS." - ) + raise RuntimeError("LoRA matched-head preflight is not PASS.") train_records = load_manifest(train_manifest_path) dev_records = load_manifest(dev_manifest_path) - train_groups = { - record["split_group_id"] - for record in train_records - } + train_groups = {record["split_group_id"] for record in train_records} - dev_groups = { - record["split_group_id"] - for record in dev_records - } + dev_groups = {record["split_group_id"] for record in dev_records} group_overlap = train_groups & dev_groups if group_overlap: - raise RuntimeError( - "Detected train/dev group overlap: " - f"{len(group_overlap)}" - ) + raise RuntimeError(f"Detected train/dev group overlap: {len(group_overlap)}") - target_attribute_ids = [ - int(value) - for value in config["target_attribute_ids"] - ] + target_attribute_ids = [int(value) for value in config["target_attribute_ids"]] target_attribute_set = set(target_attribute_ids) - if len(target_attribute_set) != len( - target_attribute_ids - ): - raise RuntimeError( - "Configured target attributes are not unique." - ) + if len(target_attribute_set) != len(target_attribute_ids): + raise RuntimeError("Configured target attributes are not unique.") target_attribute_names = { int(attribute_id): str(attribute_name) - for attribute_id, attribute_name in config[ - "target_attribute_names" - ].items() + for attribute_id, attribute_name in config["target_attribute_names"].items() } - train_attribute_ids = { - int(record["attribute_id"]) - for record in train_records - } + train_attribute_ids = {int(record["attribute_id"]) for record in train_records} - dev_attribute_ids = { - int(record["attribute_id"]) - for record in dev_records - } + dev_attribute_ids = {int(record["attribute_id"]) for record in dev_records} if train_attribute_ids != target_attribute_set: - raise RuntimeError( - "Train labels do not match configured classes." - ) + raise RuntimeError("Train labels do not match configured classes.") if dev_attribute_ids != target_attribute_set: - raise RuntimeError( - "Dev labels do not match configured classes." - ) + raise RuntimeError("Dev labels do not match configured classes.") class_index = { - attribute_id: index - for index, attribute_id in enumerate( - target_attribute_ids - ) + attribute_id: index for index, attribute_id in enumerate(target_attribute_ids) } y_train = np.asarray( - [ - class_index[int(record["attribute_id"])] - for record in train_records - ], + [class_index[int(record["attribute_id"])] for record in train_records], dtype=np.int64, ) y_dev = np.asarray( - [ - class_index[int(record["attribute_id"])] - for record in dev_records - ], + [class_index[int(record["attribute_id"])] for record in dev_records], dtype=np.int64, ) @@ -1565,24 +1331,15 @@ def main() -> None: device=device, ) - raw_audit_targets = lora_audit[ - "recommended_lora_scope" - ]["full_target_module_paths"] + raw_audit_targets = lora_audit["recommended_lora_scope"]["full_target_module_paths"] lora_target_modules = sorted( - path - for path in raw_audit_targets - if ENCODER_SELF_ATTN_PATTERN.fullmatch(path) + path for path in raw_audit_targets if ENCODER_SELF_ATTN_PATTERN.fullmatch(path) ) - excluded_targets = sorted( - set(raw_audit_targets) - - set(lora_target_modules) - ) + excluded_targets = sorted(set(raw_audit_targets) - set(lora_target_modules)) - expected_target_count = int( - config["lora"]["expected_target_module_count"] - ) + expected_target_count = int(config["lora"]["expected_target_module_count"]) if len(lora_target_modules) != expected_target_count: raise RuntimeError( @@ -1591,25 +1348,14 @@ def main() -> None: f"{expected_target_count}" ) - if excluded_targets != [ - str(config["lora"]["excluded_module"]) - ]: - raise RuntimeError( - "Unexpected LoRA excluded target set: " - f"{excluded_targets}" - ) + if excluded_targets != [str(config["lora"]["excluded_module"])]: + raise RuntimeError(f"Unexpected LoRA excluded target set: {excluded_targets}") - padding_fraction = float( - hog_config["crop"]["padding_fraction"] - ) + padding_fraction = float(hog_config["crop"]["padding_fraction"]) - if abs( - padding_fraction - - float(config["crop"]["padding_fraction"]) - ) > 1e-12: + if abs(padding_fraction - float(config["crop"]["padding_fraction"])) > 1e-12: raise RuntimeError( - "Crop padding mismatch between HOG and " - "matched-head configuration." + "Crop padding mismatch between HOG and matched-head configuration." ) image_processor = AutoImageProcessor.from_pretrained( @@ -1618,9 +1364,7 @@ def main() -> None: use_fast=False, ) - set_seed( - int(config["head"]["initialization_seed"]) - ) + set_seed(int(config["head"]["initialization_seed"])) template_head = torch.nn.Linear( int(config["head"]["input_dimension"]), @@ -1642,23 +1386,14 @@ def main() -> None: print(f"GROUP_OVERLAP_COUNT: {len(group_overlap)}") print(f"MODEL_ID: {args.model_id}") print(f"DEVICE: {device}") - print( - "PHYSICAL_BATCH_SIZE: " - f"{config['training']['physical_batch_size']}" - ) + print(f"PHYSICAL_BATCH_SIZE: {config['training']['physical_batch_size']}") print( "GRADIENT_ACCUMULATION_STEPS: " f"{config['training']['gradient_accumulation_steps']}" ) - print( - "EFFECTIVE_BATCH_SIZE: " - f"{config['training']['effective_batch_size']}" - ) + print(f"EFFECTIVE_BATCH_SIZE: {config['training']['effective_batch_size']}") print(f"EPOCHS: {config['training']['epochs']}") - print( - "LORA_TARGET_MODULES: " - f"{len(lora_target_modules)}" - ) + print(f"LORA_TARGET_MODULES: {len(lora_target_modules)}") frozen_result = run_arm( arm_name="frozen_head", @@ -1710,10 +1445,7 @@ def main() -> None: metric: { "frozen_head": frozen_metrics[metric], "lora": lora_metrics[metric], - "lora_minus_frozen_head": ( - lora_metrics[metric] - - frozen_metrics[metric] - ), + "lora_minus_frozen_head": (lora_metrics[metric] - frozen_metrics[metric]), } for metric in ( "accuracy", @@ -1728,61 +1460,31 @@ def main() -> None: } qualification_checks = { - "source_manifest_pass": ( - manifest_summary.get("status") == "PASS" - ), + "source_manifest_pass": (manifest_summary.get("status") == "PASS"), "group_disjoint_pass": len(group_overlap) == 0, - "lora_target_audit_pass": ( - lora_audit.get("status") == "PASS" - ), - "lora_preflight_pass": ( - lora_preflight.get("status") == "PASS" - ), - "frozen_arm_pass": ( - frozen_summary.get("status") == "PASS" - ), - "lora_arm_pass": ( - lora_summary.get("status") == "PASS" - ), + "lora_target_audit_pass": (lora_audit.get("status") == "PASS"), + "lora_preflight_pass": (lora_preflight.get("status") == "PASS"), + "frozen_arm_pass": (frozen_summary.get("status") == "PASS"), + "lora_arm_pass": (lora_summary.get("status") == "PASS"), "same_head_parameter_count": ( - frozen_summary["parameter_counts"][ - "classification_head" - ] - == lora_summary["parameter_counts"][ - "classification_head" - ] + frozen_summary["parameter_counts"]["classification_head"] + == lora_summary["parameter_counts"]["classification_head"] ), "lora_parameter_count_matches_config": ( lora_summary["parameter_counts"]["lora_adapter"] - == int( - config["lora"][ - "expected_adapter_parameter_count" - ] - ) + == int(config["lora"]["expected_adapter_parameter_count"]) ), "finite_comparison_macro_f1": bool( - np.isfinite( - comparison["macro_f1"][ - "lora_minus_frozen_head" - ] - ) + np.isfinite(comparison["macro_f1"]["lora_minus_frozen_head"]) ), } - status = ( - "PASS" - if all(qualification_checks.values()) - else "REVIEW" - ) + status = "PASS" if all(qualification_checks.values()) else "REVIEW" summary = { "status": status, - "schema_version": ( - "siglip2_lora_matched_head_development_run_v1" - ), - "created_utc": datetime.now( - timezone.utc - ).isoformat(), + "schema_version": ("siglip2_lora_matched_head_development_run_v1"), + "created_utc": datetime.now(UTC).isoformat(), "selection_scope": "train_only_development", "model_id": args.model_id, "environment": { @@ -1796,8 +1498,7 @@ def main() -> None: "device": str(device), "gpu_name": torch.cuda.get_device_name(0), "gpu_peak_memory_allocated_mib": round( - torch.cuda.max_memory_allocated(device) - / (1024 ** 2), + torch.cuda.max_memory_allocated(device) / (1024**2), 2, ), }, @@ -1809,24 +1510,12 @@ def main() -> None: "lora_target_audit": str(lora_audit_path), "lora_preflight": str(lora_preflight_path), "config": str(config_path), - "train_manifest_sha256": sha256_file( - train_manifest_path - ), - "dev_manifest_sha256": sha256_file( - dev_manifest_path - ), - "manifest_summary_sha256": sha256_file( - manifest_summary_path - ), - "hog_config_sha256": sha256_file( - hog_config_path - ), - "lora_target_audit_sha256": sha256_file( - lora_audit_path - ), - "lora_preflight_sha256": sha256_file( - lora_preflight_path - ), + "train_manifest_sha256": sha256_file(train_manifest_path), + "dev_manifest_sha256": sha256_file(dev_manifest_path), + "manifest_summary_sha256": sha256_file(manifest_summary_path), + "hog_config_sha256": sha256_file(hog_config_path), + "lora_target_audit_sha256": sha256_file(lora_audit_path), + "lora_preflight_sha256": sha256_file(lora_preflight_path), "config_sha256": sha256_file(config_path), }, "data": { @@ -1855,46 +1544,22 @@ def main() -> None: print("") print("===== SIGLIP2 MATCHED-HEAD DEVELOPMENT SUMMARY =====") print(f"STATUS: {status}") - print( - "FROZEN_SELECTED_EPOCH: " - f"{frozen_summary['selected_epoch']}" - ) - print( - "LORA_SELECTED_EPOCH: " - f"{lora_summary['selected_epoch']}" - ) - print( - "FROZEN_MACRO_F1: " - f"{frozen_metrics['macro_f1']:.6f}" - ) - print( - "LORA_MACRO_F1: " - f"{lora_metrics['macro_f1']:.6f}" - ) + print(f"FROZEN_SELECTED_EPOCH: {frozen_summary['selected_epoch']}") + print(f"LORA_SELECTED_EPOCH: {lora_summary['selected_epoch']}") + print(f"FROZEN_MACRO_F1: {frozen_metrics['macro_f1']:.6f}") + print(f"LORA_MACRO_F1: {lora_metrics['macro_f1']:.6f}") print( "LORA_MINUS_FROZEN_MACRO_F1: " f"{comparison['macro_f1']['lora_minus_frozen_head']:.6f}" ) - print( - "FROZEN_BALANCED_ACCURACY: " - f"{frozen_metrics['balanced_accuracy']:.6f}" - ) - print( - "LORA_BALANCED_ACCURACY: " - f"{lora_metrics['balanced_accuracy']:.6f}" - ) + print(f"FROZEN_BALANCED_ACCURACY: {frozen_metrics['balanced_accuracy']:.6f}") + print(f"LORA_BALANCED_ACCURACY: {lora_metrics['balanced_accuracy']:.6f}") print( "LORA_MINUS_FROZEN_BALANCED_ACCURACY: " f"{comparison['balanced_accuracy']['lora_minus_frozen_head']:.6f}" ) - print( - "FROZEN_TOP_LABEL_ECE: " - f"{frozen_metrics['top_label_ece']:.6f}" - ) - print( - "LORA_TOP_LABEL_ECE: " - f"{lora_metrics['top_label_ece']:.6f}" - ) + print(f"FROZEN_TOP_LABEL_ECE: {frozen_metrics['top_label_ece']:.6f}") + print(f"LORA_TOP_LABEL_ECE: {lora_metrics['top_label_ece']:.6f}") print( "LORA_MINUS_FROZEN_TOP_LABEL_ECE: " f"{comparison['top_label_ece']['lora_minus_frozen_head']:.6f}" @@ -1910,4 +1575,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/run_siglip2_matched_head_arm_v2_bf16.py b/scripts/run_siglip2_matched_head_arm_v2_bf16.py index 448a008..818f120 100644 --- a/scripts/run_siglip2_matched_head_arm_v2_bf16.py +++ b/scripts/run_siglip2_matched_head_arm_v2_bf16.py @@ -12,7 +12,7 @@ import sys import time from collections import Counter -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path import numpy as np @@ -21,8 +21,8 @@ import torch import torch.nn.functional as F import transformers -from PIL import Image from peft import LoraConfig, get_peft_model +from PIL import Image from sklearn.metrics import ( accuracy_score, balanced_accuracy_score, @@ -33,7 +33,6 @@ ) from transformers import AutoImageProcessor, AutoModel - ENCODER_SELF_ATTN_PATTERN = re.compile( r"^vision_model\.encoder\.layers\.\d+\.self_attn\." r"(q_proj|k_proj|v_proj|out_proj)$" @@ -95,9 +94,7 @@ def load_manifest(path: Path) -> list[dict[str, str]]: missing = sorted(required - set(rows[0])) if missing: - raise RuntimeError( - f"Manifest missing required columns: {missing}" - ) + raise RuntimeError(f"Manifest missing required columns: {missing}") return rows @@ -118,8 +115,7 @@ def crop_from_record( if not image_path.is_file(): raise FileNotFoundError( - "Missing image for annotation_id=" - f"{record['annotation_id']}: {image_path}" + f"Missing image for annotation_id={record['annotation_id']}: {image_path}" ) x = float(record["bbox_x"]) @@ -129,8 +125,7 @@ def crop_from_record( if width <= 0.0 or height <= 0.0: raise ValueError( - "Non-positive bbox for annotation_id=" - f"{record['annotation_id']}" + f"Non-positive bbox for annotation_id={record['annotation_id']}" ) with Image.open(image_path) as source: @@ -153,8 +148,7 @@ def crop_from_record( if right <= left or bottom <= top: raise ValueError( - "Degenerate crop for annotation_id=" - f"{record['annotation_id']}" + f"Degenerate crop for annotation_id={record['annotation_id']}" ) return image.crop((left, top, right, bottom)) @@ -166,10 +160,7 @@ def prepare_inputs( device: torch.device, padding_fraction: float, ) -> dict[str, torch.Tensor]: - crops = [ - crop_from_record(record, padding_fraction) - for record in records - ] + crops = [crop_from_record(record, padding_fraction) for record in records] processor_outputs = image_processor( images=crops, @@ -196,9 +187,7 @@ def cpu_state_dict(module: torch.nn.Module) -> dict: def count_parameters(module: torch.nn.Module) -> int: - return int( - sum(parameter.numel() for parameter in module.parameters()) - ) + return int(sum(parameter.numel() for parameter in module.parameters())) def trainable_named_parameters( @@ -249,10 +238,7 @@ def gradient_health( "missing_gradient_names": missing, "nonfinite_gradient_names": nonfinite, "max_abs_gradient": max_abs_gradient, - "all_present_and_finite": ( - len(missing) == 0 - and len(nonfinite) == 0 - ), + "all_present_and_finite": (len(missing) == 0 and len(nonfinite) == 0), } @@ -274,15 +260,9 @@ def top_label_ece( upper = float(edges[index + 1]) if index == bin_count - 1: - mask = ( - (confidence >= lower) - & (confidence <= upper) - ) + mask = (confidence >= lower) & (confidence <= upper) else: - mask = ( - (confidence >= lower) - & (confidence < upper) - ) + mask = (confidence >= lower) & (confidence < upper) count = int(mask.sum()) @@ -317,22 +297,14 @@ def build_class_weights( counts = np.bincount(labels, minlength=class_count) if np.any(counts <= 0): - raise RuntimeError( - "Every target class must appear in train labels." - ) + raise RuntimeError("Every target class must appear in train labels.") total = float(labels.shape[0]) - raw = total / ( - float(class_count) - * counts.astype(np.float64) - ) + raw = total / (float(class_count) * counts.astype(np.float64)) return ( torch.tensor(raw, dtype=torch.float32, device=device), - { - str(index): float(raw[index]) - for index in range(class_count) - }, + {str(index): float(raw[index]) for index in range(class_count)}, ) @@ -393,22 +365,13 @@ def evaluate( ) if not torch.isfinite(loss).item(): - raise FloatingPointError( - "Non-finite evaluation loss." - ) + raise FloatingPointError("Non-finite evaluation loss.") - logits_chunks.append( - logits.detach().float().cpu().numpy() - ) + logits_chunks.append(logits.detach().float().cpu().numpy()) - target_chunks.append( - batch_targets.detach().cpu().numpy() - ) + target_chunks.append(batch_targets.detach().cpu().numpy()) - weighted_loss_sum += ( - float(loss.detach().cpu().item()) - * len(batch_records) - ) + weighted_loss_sum += float(loss.detach().cpu().item()) * len(batch_records) observed_count += len(batch_records) @@ -419,10 +382,7 @@ def evaluate( len(records), len(target_attribute_ids), ): - raise RuntimeError( - "Unexpected evaluation logit shape: " - f"{logits_all.shape}" - ) + raise RuntimeError(f"Unexpected evaluation logit shape: {logits_all.shape}") shifted = logits_all - logits_all.max( axis=1, @@ -436,19 +396,15 @@ def evaluate( ) if not np.isfinite(probabilities).all(): - raise FloatingPointError( - "Non-finite evaluation probabilities." - ) + raise FloatingPointError("Non-finite evaluation probabilities.") predicted_indices = probabilities.argmax(axis=1) - precision, recall, f1, support = ( - precision_recall_fscore_support( - y_true, - predicted_indices, - labels=list(range(len(target_attribute_ids))), - zero_division=0, - ) + precision, recall, f1, support = precision_recall_fscore_support( + y_true, + predicted_indices, + labels=list(range(len(target_attribute_ids))), + zero_division=0, ) confusion = confusion_matrix( @@ -469,12 +425,8 @@ def evaluate( "evaluation_weighted_cross_entropy": ( weighted_loss_sum / float(observed_count) ), - "accuracy": float( - accuracy_score(y_true, predicted_indices) - ), - "balanced_accuracy": float( - balanced_accuracy_score(y_true, predicted_indices) - ), + "accuracy": float(accuracy_score(y_true, predicted_indices)), + "balanced_accuracy": float(balanced_accuracy_score(y_true, predicted_indices)), "macro_f1": float( f1_score( y_true, @@ -507,7 +459,8 @@ def evaluate( len(target_attribute_ids), dtype=np.float64, )[y_true] - ) ** 2, + ) + ** 2, axis=1, ) ) @@ -525,15 +478,11 @@ def evaluate( per_class_rows = [] - for class_index, attribute_id in enumerate( - target_attribute_ids - ): + for class_index, attribute_id in enumerate(target_attribute_ids): per_class_rows.append( { "attribute_id": int(attribute_id), - "attribute_name": target_attribute_names[ - int(attribute_id) - ], + "attribute_name": target_attribute_names[int(attribute_id)], "precision": float(precision[class_index]), "recall": float(recall[class_index]), "f1": float(f1[class_index]), @@ -544,9 +493,7 @@ def evaluate( prediction_rows = [] for row_index, record in enumerate(records): - predicted_attribute_id = target_attribute_ids[ - int(predicted_indices[row_index]) - ] + predicted_attribute_id = target_attribute_ids[int(predicted_indices[row_index])] row = { "annotation_id": record["annotation_id"], @@ -555,29 +502,16 @@ def evaluate( "image_path": record["image_path"], "true_attribute_id": int(record["attribute_id"]), "true_attribute_name": record["attribute_name"], - "predicted_attribute_id": int( - predicted_attribute_id - ), + "predicted_attribute_id": int(predicted_attribute_id), "predicted_attribute_name": ( - target_attribute_names[ - predicted_attribute_id - ] - ), - "confidence": float( - probabilities[row_index].max() - ), - "correct": bool( - predicted_attribute_id - == int(record["attribute_id"]) + target_attribute_names[predicted_attribute_id] ), + "confidence": float(probabilities[row_index].max()), + "correct": bool(predicted_attribute_id == int(record["attribute_id"])), } - for class_position, attribute_id in enumerate( - target_attribute_ids - ): - row[ - f"probability_attribute_{attribute_id}" - ] = float( + for class_position, attribute_id in enumerate(target_attribute_ids): + row[f"probability_attribute_{attribute_id}"] = float( probabilities[row_index, class_position] ) @@ -585,22 +519,14 @@ def evaluate( confusion_rows = [] - for true_index, true_attribute_id in enumerate( - target_attribute_ids - ): + for true_index, true_attribute_id in enumerate(target_attribute_ids): row = { "true_attribute_id": int(true_attribute_id), - "true_attribute_name": target_attribute_names[ - int(true_attribute_id) - ], + "true_attribute_name": target_attribute_names[int(true_attribute_id)], } - for predicted_index, predicted_attribute_id in enumerate( - target_attribute_ids - ): - row[ - f"predicted_attribute_{predicted_attribute_id}" - ] = int( + for predicted_index, predicted_attribute_id in enumerate(target_attribute_ids): + row[f"predicted_attribute_{predicted_attribute_id}"] = int( confusion[true_index, predicted_index] ) @@ -641,18 +567,14 @@ def train_epoch( head.train() - order = np.random.default_rng( - shuffle_seed + epoch - ).permutation(len(train_records)) + order = np.random.default_rng(shuffle_seed + epoch).permutation(len(train_records)) total_loss = 0.0 total_samples = 0 optimizer_steps = 0 max_abs_gradient = 0.0 - effective_group_size = ( - physical_batch_size * accumulation_steps - ) + effective_group_size = physical_batch_size * accumulation_steps for group_start in range( 0, @@ -660,7 +582,7 @@ def train_epoch( effective_group_size, ): group_indices = order[ - group_start:min( + group_start : min( group_start + effective_group_size, len(train_records), ) @@ -668,7 +590,7 @@ def train_epoch( micro_batches = [ group_indices[ - start:min( + start : min( start + physical_batch_size, len(group_indices), ) @@ -683,10 +605,7 @@ def train_epoch( optimizer.zero_grad(set_to_none=True) for micro_indices in micro_batches: - batch_records = [ - train_records[int(index)] - for index in micro_indices - ] + batch_records = [train_records[int(index)] for index in micro_indices] batch_targets = torch.tensor( train_labels[micro_indices], @@ -739,24 +658,15 @@ def train_epoch( f"group_start={group_start}" ) - ( - raw_loss / float(len(micro_batches)) - ).backward() + (raw_loss / float(len(micro_batches))).backward() - total_loss += ( - float(raw_loss.detach().cpu().item()) - * len(batch_records) - ) + total_loss += float(raw_loss.detach().cpu().item()) * len(batch_records) total_samples += len(batch_records) trainable = trainable_named_parameters( head=head, - lora_model=( - None - if arm_name == "frozen_head" - else mode_module - ), + lora_model=(None if arm_name == "frozen_head" else mode_module), ) health = gradient_health(trainable) @@ -788,17 +698,14 @@ def train_epoch( if not torch.isfinite(clip_value).item(): raise FloatingPointError( - "Non-finite gradient norm " - f"arm={arm_name} epoch={epoch}" + f"Non-finite gradient norm arm={arm_name} epoch={epoch}" ) optimizer.step() optimizer_steps += 1 return { - "train_weighted_cross_entropy": ( - total_loss / float(total_samples) - ), + "train_weighted_cross_entropy": (total_loss / float(total_samples)), "optimizer_steps": int(optimizer_steps), "max_abs_gradient": float(max_abs_gradient), } @@ -811,18 +718,11 @@ def better_candidate( if current_best is None: return True - if ( - candidate["macro_f1"] - > current_best["macro_f1"] + 1e-12 - ): + if candidate["macro_f1"] > current_best["macro_f1"] + 1e-12: return True return ( - abs( - candidate["macro_f1"] - - current_best["macro_f1"] - ) - <= 1e-12 + abs(candidate["macro_f1"] - current_best["macro_f1"]) <= 1e-12 and candidate["negative_log_likelihood"] < current_best["negative_log_likelihood"] ) @@ -866,11 +766,7 @@ def save_checkpoint( "arm_name": arm_name, "epoch": int(epoch), "head_state_path": str(head_path), - "adapter_directory": ( - None - if adapter_dir is None - else str(adapter_dir) - ), + "adapter_directory": (None if adapter_dir is None else str(adapter_dir)), "selected_metrics": metrics, } @@ -878,11 +774,7 @@ def save_checkpoint( return { "head_state_path": str(head_path), - "adapter_directory": ( - None - if adapter_dir is None - else str(adapter_dir) - ), + "adapter_directory": (None if adapter_dir is None else str(adapter_dir)), "checkpoint_metadata_json": str(metadata_path), } @@ -910,9 +802,7 @@ def main() -> None: train_manifest_path = Path(args.train_manifest).resolve() dev_manifest_path = Path(args.dev_manifest).resolve() - manifest_summary_path = Path( - args.manifest_summary - ).resolve() + manifest_summary_path = Path(args.manifest_summary).resolve() hog_config_path = Path(args.hog_config).resolve() audit_path = Path(args.lora_audit).resolve() bf16_preflight_path = Path(args.bf16_preflight).resolve() @@ -930,29 +820,21 @@ def main() -> None: config_path, ): if not path.is_file(): - raise FileNotFoundError( - f"Required artifact missing: {path}" - ) + raise FileNotFoundError(f"Required artifact missing: {path}") if not run_root.is_dir(): - raise FileNotFoundError( - f"Run root does not exist: {run_root}" - ) + raise FileNotFoundError(f"Run root does not exist: {run_root}") arm_dir = run_root / args.arm if arm_dir.exists(): - raise FileExistsError( - f"Refusing to overwrite arm directory: {arm_dir}" - ) + raise FileExistsError(f"Refusing to overwrite arm directory: {arm_dir}") if not torch.cuda.is_available(): raise RuntimeError("CUDA is unavailable.") if not torch.cuda.is_bf16_supported(): - raise RuntimeError( - "CUDA device does not report bf16 support." - ) + raise RuntimeError("CUDA device does not report bf16 support.") manifest_summary = load_json(manifest_summary_path) hog_config = load_json(hog_config_path) @@ -982,9 +864,7 @@ def main() -> None: "frozen_gradients_finite_and_present", False, ): - raise RuntimeError( - "Frozen BF16 gradient preflight was not qualified." - ) + raise RuntimeError("Frozen BF16 gradient preflight was not qualified.") if not bf16_preflight.get( "checks", @@ -993,54 +873,32 @@ def main() -> None: "lora_gradients_finite_and_present", False, ): - raise RuntimeError( - "LoRA BF16 gradient preflight was not qualified." - ) + raise RuntimeError("LoRA BF16 gradient preflight was not qualified.") train_records = load_manifest(train_manifest_path) dev_records = load_manifest(dev_manifest_path) - train_groups = { - row["split_group_id"] - for row in train_records - } + train_groups = {row["split_group_id"] for row in train_records} - dev_groups = { - row["split_group_id"] - for row in dev_records - } + dev_groups = {row["split_group_id"] for row in dev_records} group_overlap = train_groups & dev_groups if group_overlap: - raise RuntimeError( - "Train/dev group overlap detected: " - f"{len(group_overlap)}" - ) + raise RuntimeError(f"Train/dev group overlap detected: {len(group_overlap)}") - target_attribute_ids = [ - int(value) - for value in config["target_attribute_ids"] - ] + target_attribute_ids = [int(value) for value in config["target_attribute_ids"]] target_attribute_names = { int(attribute_id): str(name) - for attribute_id, name in config[ - "target_attribute_names" - ].items() + for attribute_id, name in config["target_attribute_names"].items() } target_attribute_set = set(target_attribute_ids) - train_attributes = { - int(row["attribute_id"]) - for row in train_records - } + train_attributes = {int(row["attribute_id"]) for row in train_records} - dev_attributes = { - int(row["attribute_id"]) - for row in dev_records - } + dev_attributes = {int(row["attribute_id"]) for row in dev_records} if train_attributes != target_attribute_set: raise RuntimeError("Train labels do not match task labels.") @@ -1049,68 +907,37 @@ def main() -> None: raise RuntimeError("Dev labels do not match task labels.") class_index = { - attribute_id: index - for index, attribute_id in enumerate( - target_attribute_ids - ) + attribute_id: index for index, attribute_id in enumerate(target_attribute_ids) } y_train = np.asarray( - [ - class_index[int(row["attribute_id"])] - for row in train_records - ], + [class_index[int(row["attribute_id"])] for row in train_records], dtype=np.int64, ) y_dev = np.asarray( - [ - class_index[int(row["attribute_id"])] - for row in dev_records - ], + [class_index[int(row["attribute_id"])] for row in dev_records], dtype=np.int64, ) - padding_fraction = float( - config["crop"]["padding_fraction"] - ) + padding_fraction = float(config["crop"]["padding_fraction"]) - if abs( - padding_fraction - - float(hog_config["crop"]["padding_fraction"]) - ) > 1e-12: - raise RuntimeError( - "HOG and matched-head crop padding do not match." - ) + if abs(padding_fraction - float(hog_config["crop"]["padding_fraction"])) > 1e-12: + raise RuntimeError("HOG and matched-head crop padding do not match.") - raw_targets = lora_audit[ - "recommended_lora_scope" - ]["full_target_module_paths"] + raw_targets = lora_audit["recommended_lora_scope"]["full_target_module_paths"] lora_targets = sorted( - path - for path in raw_targets - if ENCODER_SELF_ATTN_PATTERN.fullmatch(path) + path for path in raw_targets if ENCODER_SELF_ATTN_PATTERN.fullmatch(path) ) - excluded_targets = sorted( - set(raw_targets) - set(lora_targets) - ) + excluded_targets = sorted(set(raw_targets) - set(lora_targets)) - if len(lora_targets) != int( - config["lora"]["expected_target_module_count"] - ): - raise RuntimeError( - "LoRA target count does not match config." - ) + if len(lora_targets) != int(config["lora"]["expected_target_module_count"]): + raise RuntimeError("LoRA target count does not match config.") - if excluded_targets != [ - str(config["lora"]["excluded_module"]) - ]: - raise RuntimeError( - "Unexpected LoRA excluded target list: " - f"{excluded_targets}" - ) + if excluded_targets != [str(config["lora"]["excluded_module"])]: + raise RuntimeError(f"Unexpected LoRA excluded target list: {excluded_targets}") device = torch.device("cuda") set_seed(int(config["training"]["seed"])) @@ -1128,9 +955,7 @@ def main() -> None: freeze_module(base_model) - set_seed( - int(config["head"]["initialization_seed"]) - ) + set_seed(int(config["head"]["initialization_seed"])) head = torch.nn.Linear( int(config["head"]["input_dimension"]), @@ -1149,12 +974,8 @@ def main() -> None: optimizer = torch.optim.AdamW( head.parameters(), - lr=float( - config["training"]["head_learning_rate"] - ), - weight_decay=float( - config["training"]["weight_decay"] - ), + lr=float(config["training"]["head_learning_rate"]), + weight_decay=float(config["training"]["weight_decay"]), ) else: lora_config = LoraConfig( @@ -1194,14 +1015,12 @@ def main() -> None: config["lora"]["expected_adapter_parameter_count"] ): raise RuntimeError( - "LoRA adapter parameter count mismatch: " - f"{adapter_parameter_count}" + f"LoRA adapter parameter count mismatch: {adapter_parameter_count}" ) if unexpected_base_trainables: raise RuntimeError( - "Unexpected base trainables: " - f"{unexpected_base_trainables}" + f"Unexpected base trainables: {unexpected_base_trainables}" ) adapter_parameters = [ @@ -1214,25 +1033,13 @@ def main() -> None: [ { "params": adapter_parameters, - "lr": float( - config["training"][ - "lora_learning_rate" - ] - ), - "weight_decay": float( - config["training"]["weight_decay"] - ), + "lr": float(config["training"]["lora_learning_rate"]), + "weight_decay": float(config["training"]["weight_decay"]), }, { "params": list(head.parameters()), - "lr": float( - config["training"][ - "head_learning_rate" - ] - ), - "weight_decay": float( - config["training"]["weight_decay"] - ), + "lr": float(config["training"]["head_learning_rate"]), + "weight_decay": float(config["training"]["weight_decay"]), }, ] ) @@ -1263,8 +1070,7 @@ def main() -> None: print("AUTOCAST_DTYPE: bfloat16", flush=True) print("GRAD_SCALER: False", flush=True) print( - "PHYSICAL_BATCH_SIZE: " - f"{config['training']['physical_batch_size']}", + f"PHYSICAL_BATCH_SIZE: {config['training']['physical_batch_size']}", flush=True, ) print( @@ -1273,8 +1079,7 @@ def main() -> None: flush=True, ) print( - "EFFECTIVE_BATCH_SIZE: " - f"{config['training']['effective_batch_size']}", + f"EFFECTIVE_BATCH_SIZE: {config['training']['effective_batch_size']}", flush=True, ) print(f"EPOCHS: {config['training']['epochs']}", flush=True) @@ -1288,12 +1093,8 @@ def main() -> None: ) epochs = int(config["training"]["epochs"]) - batch_size = int( - config["training"]["physical_batch_size"] - ) - accumulation_steps = int( - config["training"]["gradient_accumulation_steps"] - ) + batch_size = int(config["training"]["physical_batch_size"]) + accumulation_steps = int(config["training"]["gradient_accumulation_steps"]) epoch_rows = [] best_metrics = None @@ -1323,9 +1124,7 @@ def main() -> None: accumulation_steps=accumulation_steps, epoch=epoch, shuffle_seed=int(config["training"]["seed"]), - max_grad_norm=float( - config["training"]["max_grad_norm"] - ), + max_grad_norm=float(config["training"]["max_grad_norm"]), ) ( @@ -1439,8 +1238,7 @@ def main() -> None: "confidence", "correct", ] + [ - f"probability_attribute_{attribute_id}" - for attribute_id in target_attribute_ids + f"probability_attribute_{attribute_id}" for attribute_id in target_attribute_ids ] write_csv( @@ -1454,7 +1252,8 @@ def main() -> None: fieldnames=[ "true_attribute_id", "true_attribute_name", - ] + [ + ] + + [ f"predicted_attribute_{attribute_id}" for attribute_id in target_attribute_ids ], @@ -1465,46 +1264,30 @@ def main() -> None: ece_path, { "top_label_ece": best_metrics["top_label_ece"], - "bin_count": int( - config["evaluation"]["ece_bins"] - ), + "bin_count": int(config["evaluation"]["ece_bins"]), "bins": best_ece, }, ) - expected_head_parameters = ( - int(config["head"]["input_dimension"]) - * int(config["head"]["output_classes"]) - + int(config["head"]["output_classes"]) - ) + expected_head_parameters = int(config["head"]["input_dimension"]) * int( + config["head"]["output_classes"] + ) + int(config["head"]["output_classes"]) checks = { - "source_manifest_pass": ( - manifest_summary.get("status") == "PASS" - ), + "source_manifest_pass": (manifest_summary.get("status") == "PASS"), "group_disjoint_pass": len(group_overlap) == 0, - "bf16_supported": bool( - torch.cuda.is_bf16_supported() - ), - "bf16_preflight_pass": ( - bf16_preflight.get("status") == "PASS" - ), + "bf16_supported": bool(torch.cuda.is_bf16_supported()), + "bf16_preflight_pass": (bf16_preflight.get("status") == "PASS"), "best_epoch_selected": best_epoch is not None, "all_epoch_optimizer_steps_positive": all( - row["optimizer_steps"] > 0 - for row in epoch_rows - ), - "finite_selected_macro_f1": bool( - np.isfinite(best_metrics["macro_f1"]) + row["optimizer_steps"] > 0 for row in epoch_rows ), + "finite_selected_macro_f1": bool(np.isfinite(best_metrics["macro_f1"])), "finite_selected_nll": bool( - np.isfinite( - best_metrics["negative_log_likelihood"] - ) + np.isfinite(best_metrics["negative_log_likelihood"]) ), "head_parameter_count_expected": ( - count_parameters(head) - == expected_head_parameters + count_parameters(head) == expected_head_parameters ), "best_prediction_count_matches_dev": ( len(best_predictions) == len(dev_records) @@ -1512,43 +1295,25 @@ def main() -> None: } if args.arm == "frozen_head": - checks["frozen_adapter_parameter_count_zero"] = ( - adapter_parameter_count == 0 - ) + checks["frozen_adapter_parameter_count_zero"] = adapter_parameter_count == 0 else: - checks["lora_target_count_matches"] = ( - len(lora_targets) - == int( - config["lora"][ - "expected_target_module_count" - ] - ) + checks["lora_target_count_matches"] = len(lora_targets) == int( + config["lora"]["expected_target_module_count"] ) - checks["lora_adapter_parameter_count_matches"] = ( - adapter_parameter_count - == int( - config["lora"][ - "expected_adapter_parameter_count" - ] - ) + checks["lora_adapter_parameter_count_matches"] = adapter_parameter_count == int( + config["lora"]["expected_adapter_parameter_count"] ) - checks["no_unexpected_base_trainables"] = ( - len(unexpected_base_trainables) == 0 - ) + checks["no_unexpected_base_trainables"] = len(unexpected_base_trainables) == 0 status = "PASS" if all(checks.values()) else "REVIEW" summary = { "status": status, - "schema_version": ( - "siglip2_matched_head_arm_v2_bf16" - ), + "schema_version": ("siglip2_matched_head_arm_v2_bf16"), "arm_name": args.arm, - "created_utc": datetime.now( - timezone.utc - ).isoformat(), + "created_utc": datetime.now(UTC).isoformat(), "selection_scope": "train_only_development", "selected_epoch": int(best_epoch), "selected_metrics": best_metrics, @@ -1556,7 +1321,7 @@ def main() -> None: "precision_policy": { "autocast_dtype": "bfloat16", "grad_scaler_used": False, - "loss_dtype": "float32" + "loss_dtype": "float32", }, "source_artifacts": { "train_manifest": str(train_manifest_path), @@ -1566,12 +1331,8 @@ def main() -> None: "lora_audit": str(audit_path), "bf16_preflight": str(bf16_preflight_path), "config": str(config_path), - "train_manifest_sha256": sha256_file( - train_manifest_path - ), - "dev_manifest_sha256": sha256_file( - dev_manifest_path - ), + "train_manifest_sha256": sha256_file(train_manifest_path), + "dev_manifest_sha256": sha256_file(dev_manifest_path), "config_sha256": sha256_file(config_path), }, "environment": { @@ -1585,8 +1346,7 @@ def main() -> None: "device": str(device), "gpu_name": torch.cuda.get_device_name(0), "gpu_peak_memory_allocated_mib": round( - torch.cuda.max_memory_allocated(device) - / (1024 ** 2), + torch.cuda.max_memory_allocated(device) / (1024**2), 2, ), }, @@ -1602,29 +1362,20 @@ def main() -> None: "parameter_counts": { "classification_head": count_parameters(head), "lora_adapter": adapter_parameter_count, - "total_trainable": ( - count_parameters(head) - + adapter_parameter_count - ), + "total_trainable": (count_parameters(head) + adapter_parameter_count), }, "lora_scope": { "selected_target_modules": ( - [] - if args.arm == "frozen_head" - else lora_targets - ), - "unexpected_base_trainables": ( - unexpected_base_trainables + [] if args.arm == "frozen_head" else lora_targets ), + "unexpected_base_trainables": (unexpected_base_trainables), }, "qualification_checks": checks, "artifacts": { "epoch_metrics_csv": str(epoch_metrics_path), "best_per_class_metrics_csv": str(per_class_path), "best_dev_predictions_csv": str(predictions_path), - "best_dev_confusion_matrix_csv": str( - confusion_path - ), + "best_dev_confusion_matrix_csv": str(confusion_path), "best_dev_ece_bins_json": str(ece_path), **best_checkpoint, }, @@ -1642,24 +1393,11 @@ def main() -> None: print(f"STATUS: {status}") print(f"ARM: {args.arm}") print(f"SELECTED_EPOCH: {best_epoch}") - print( - f"MACRO_F1: {best_metrics['macro_f1']:.6f}" - ) - print( - "BALANCED_ACCURACY: " - f"{best_metrics['balanced_accuracy']:.6f}" - ) - print( - f"TOP_LABEL_ECE: {best_metrics['top_label_ece']:.6f}" - ) - print( - "NEGATIVE_LOG_LIKELIHOOD: " - f"{best_metrics['negative_log_likelihood']:.6f}" - ) - print( - "TOP_2_ACCURACY: " - f"{best_metrics['top_2_accuracy']:.6f}" - ) + print(f"MACRO_F1: {best_metrics['macro_f1']:.6f}") + print(f"BALANCED_ACCURACY: {best_metrics['balanced_accuracy']:.6f}") + print(f"TOP_LABEL_ECE: {best_metrics['top_label_ece']:.6f}") + print(f"NEGATIVE_LOG_LIKELIHOOD: {best_metrics['negative_log_likelihood']:.6f}") + print(f"TOP_2_ACCURACY: {best_metrics['top_2_accuracy']:.6f}") print( "GPU_PEAK_MEMORY_ALLOCATED_MIB: " f"{summary['environment']['gpu_peak_memory_allocated_mib']:.2f}" diff --git a/scripts/smoke_siglip2_image_forward.py b/scripts/smoke_siglip2_image_forward.py index 0b86de2..c08a53a 100644 --- a/scripts/smoke_siglip2_image_forward.py +++ b/scripts/smoke_siglip2_image_forward.py @@ -9,8 +9,8 @@ import numpy as np import torch -from PIL import Image import transformers +from PIL import Image from transformers import AutoImageProcessor, AutoModel @@ -40,8 +40,7 @@ def crop_from_record( if width <= 0.0 or height <= 0.0: raise ValueError( - "Non-positive bbox for " - f"annotation_id={record['annotation_id']}" + f"Non-positive bbox for annotation_id={record['annotation_id']}" ) with Image.open(image_path) as source: @@ -64,8 +63,7 @@ def crop_from_record( if right <= left or bottom <= top: raise ValueError( - "Degenerate crop for " - f"annotation_id={record['annotation_id']}" + f"Degenerate crop for annotation_id={record['annotation_id']}" ) return image.crop((left, top, right, bottom)) @@ -95,17 +93,13 @@ def main() -> None: with hog_config_path.open("r", encoding="utf-8") as handle: hog_config = json.load(handle) - padding_fraction = float( - hog_config["crop"]["padding_fraction"] - ) + padding_fraction = float(hog_config["crop"]["padding_fraction"]) record = load_first_record(manifest_path) crop = crop_from_record(record, padding_fraction) if not torch.cuda.is_available(): - raise RuntimeError( - "CUDA is unavailable in the active venv." - ) + raise RuntimeError("CUDA is unavailable in the active venv.") device = torch.device("cuda") torch.cuda.empty_cache() @@ -144,22 +138,13 @@ def main() -> None: torch.cuda.synchronize(device) - features = ( - image_features.detach() - .float() - .cpu() - .numpy() - ) + features = image_features.detach().float().cpu().numpy() if features.ndim != 2 or features.shape[0] != 1: - raise RuntimeError( - f"Unexpected embedding shape: {features.shape}" - ) + raise RuntimeError(f"Unexpected embedding shape: {features.shape}") if not np.isfinite(features).all(): - raise RuntimeError( - "Embedding contains non-finite values." - ) + raise RuntimeError("Embedding contains non-finite values.") payload = { "status": "PASS", @@ -176,11 +161,11 @@ def main() -> None: "device": str(device), "gpu_name": torch.cuda.get_device_name(0), "gpu_total_memory_mib": round( - torch.cuda.get_device_properties(0).total_memory / (1024 ** 2), + torch.cuda.get_device_properties(0).total_memory / (1024**2), 2, ), "gpu_peak_memory_allocated_mib": round( - torch.cuda.max_memory_allocated(device) / (1024 ** 2), + torch.cuda.max_memory_allocated(device) / (1024**2), 2, ), "model_parameter_count": int( @@ -205,9 +190,7 @@ def main() -> None: "feature_shape": list(features.shape), "feature_dtype": str(features.dtype), "feature_l2_norm": float(np.linalg.norm(features[0])), - "finite_feature_values": bool( - np.isfinite(features).all() - ), + "finite_feature_values": bool(np.isfinite(features).all()), "claim_boundary": ( "Single-crop frozen SigLIP2 image-forward smoke test only. " "No full-manifest feature extraction, classifier fitting, " @@ -224,4 +207,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/validate_documentation.py b/scripts/validate_documentation.py index 29aff23..8c51d74 100644 --- a/scripts/validate_documentation.py +++ b/scripts/validate_documentation.py @@ -1,8 +1,9 @@ """Validate public documentation claims and links without model or dataset access.""" + from __future__ import annotations -from pathlib import Path import sys +from pathlib import Path ROOT = Path(__file__).resolve().parents[1] diff --git a/scripts/verify_hog_environment.py b/scripts/verify_hog_environment.py index 5694849..5d74678 100644 --- a/scripts/verify_hog_environment.py +++ b/scripts/verify_hog_environment.py @@ -7,8 +7,8 @@ import numpy import PIL -import sklearn import skimage +import sklearn def main() -> None: @@ -41,4 +41,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/write_hog_environment_preflight.py b/scripts/write_hog_environment_preflight.py index 8355b79..b868ab1 100644 --- a/scripts/write_hog_environment_preflight.py +++ b/scripts/write_hog_environment_preflight.py @@ -7,8 +7,8 @@ import numpy import PIL -import sklearn import skimage +import sklearn def main() -> None: @@ -40,4 +40,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main()