{ "cells": [ { "cell_type": "markdown", "id": "2f04eee0-5928-4e74-a754-6dc2e528810c", "metadata": {}, "source": [ "# SystemsAge" ] }, { "cell_type": "markdown", "id": "a3f514a3-772c-4a14-afdf-5a8376851ff4", "metadata": {}, "source": [ "## Index\n", "1. [Instantiate model class](#Instantiate-model-class)\n", "2. [Define clock metadata](#Define-clock-metadata)\n", "3. [Download clock dependencies](#Download-clock-dependencies)\n", "5. [Load features](#Load-features)\n", "6. [Load weights into base model](#Load-weights-into-base-model)\n", "8. [Load preprocess and postprocess objects](#Load-preprocess-and-postprocess-objects)\n", "10. [Basic test](#Basic-test)\n", "11. [Save torch model](#Save-torch-model)\n", "12. [Clear directory](#Clear-directory)" ] }, { "cell_type": "markdown", "id": "d95fafdc-643a-40ea-a689-200bd132e90c", "metadata": {}, "source": [ "Let's first import some packages:" ] }, { "cell_type": "code", "execution_count": 1, "id": "4adfb4de-cd79-4913-a1af-9e23e9e236c9", "metadata": { "execution": { "iopub.execute_input": "2024-03-05T21:01:00.984260Z", "iopub.status.busy": "2024-03-05T21:01:00.983856Z", "iopub.status.idle": "2024-03-05T21:01:02.504320Z", "shell.execute_reply": "2024-03-05T21:01:02.504017Z" } }, "outputs": [], "source": [ "import os\n", "import inspect\n", "import sys\n", "import subprocess\n", "import shutil\n", "import json\n", "import torch\n", "import pandas as pd\n", "import pyaging as pya\n", "from pathlib import Path\n", "import numpy as np\n", "import torch.nn as nn" ] }, { "cell_type": "markdown", "id": "145082e5-ced4-47ae-88c0-cb69773e3c5a", "metadata": {}, "source": [ "## Instantiate model class" ] }, { "cell_type": "code", "execution_count": 2, "id": "8aa77372-7ed3-4da7-abc9-d30372106139", "metadata": { "execution": { "iopub.execute_input": "2024-03-05T21:01:02.506190Z", "iopub.status.busy": "2024-03-05T21:01:02.506029Z", "iopub.status.idle": "2024-03-05T21:01:02.515402Z", "shell.execute_reply": "2024-03-05T21:01:02.515149Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "class SystemsAge(SystemsAgeBase):\n", " def __init__(self):\n", " super().__init__()\n", " self.prediction_index = -1\n", "\n" ] } ], "source": [ "def print_entire_class(cls):\n", " source = inspect.getsource(cls)\n", " print(source)\n", "\n", "print_entire_class(pya.models.SystemsAge)" ] }, { "cell_type": "code", "execution_count": 3, "id": "78536494-f1d9-44de-8583-c89a310d2307", "metadata": { "execution": { "iopub.execute_input": "2024-03-05T21:01:02.516852Z", "iopub.status.busy": "2024-03-05T21:01:02.516771Z", "iopub.status.idle": "2024-03-05T21:01:02.518516Z", "shell.execute_reply": "2024-03-05T21:01:02.518259Z" } }, "outputs": [], "source": [ "model = pya.models.SystemsAge()" ] }, { "cell_type": "markdown", "id": "51f8615e-01fa-4aa5-b196-3ee2b35d261c", "metadata": {}, "source": [ "## Define clock metadata" ] }, { "cell_type": "code", "execution_count": 4, "id": "6601da9e-8adc-44ee-9308-75e3cd31b816", "metadata": { "execution": { "iopub.execute_input": "2024-03-05T21:01:02.520071Z", "iopub.status.busy": "2024-03-05T21:01:02.519980Z", "iopub.status.idle": "2024-03-05T21:01:02.522034Z", "shell.execute_reply": "2024-03-05T21:01:02.521798Z" } }, "outputs": [], "source": [ "model.metadata[\"clock_name\"] = \"systemsage\"\n", "model.metadata[\"data_type\"] = \"DNA methylation\" # Paper: blood DNA methylation data\n", "model.metadata[\"species\"] = \"Homo sapiens\" # Paper: participants from the HRS and FHS human cohorts\n", "model.metadata[\"year\"] = 2025\n", "model.metadata[\"approved_by_author\"] = \"⌛\"\n", "model.metadata[\"citation\"] = \"Sehgal, R., Markov, Y., Qin, C., et al. (2025). Systems Age: a single blood methylation test to quantify aging heterogeneity across 11 physiological systems. Nature Aging, 5, 1880–1896.\"\n", "model.metadata[\"doi\"] = \"https://doi.org/10.1038/s43587-025-00958-3\"\n", "model.metadata[\"notes\"] = \"Composite Systems Age integrates the 11 mortality-associated physiological-system scores plus a DNAm chronological-age prediction through PCA and Cox elastic-net regression, then rescales the result to an age-like value.\"\n", "model.metadata[\"research_only\"] = True\n", "model.metadata[\"tissue\"] = [\"whole blood\"] # Paper: single blood draw; HRS and FHS blood DNAm\n", "model.metadata[\"predicts\"] = [\"multisystem biological age\"] # Paper: composite Systems Age score to capture overall multisystem aging\n", "model.metadata[\"training_target\"] = [\"mortality\"] # Paper: Cox elastic net model predicting mortality\n", "model.metadata[\"unit\"] = [\"years\"] # Paper: system_ages[, i] <- system_ages[, i] / 12\n", "model.metadata[\"model_type\"] = \"PCA + elastic net regression\" # Paper: PCA on system scores followed by Cox elastic net\n", "model.metadata[\"platform\"] = [\"Illumina 450K\", \"Illumina EPIC\"] # Paper: 125,175 CpGs present across training and validation datasets using both Illumina 450K and EPIC arrays\n", "model.metadata[\"population\"] = \"older adults\" # Paper: HRS clinical training ages 51–100; FHS Offspring used for mortality training and both FHS cohorts for scaling\n", "model.metadata[\"journal\"] = \"Nature Aging\"\n", "model.metadata[\"last_author\"] = \"Morgan Levine\"\n", "model.metadata[\"n_features\"] = 125175\n", "model.metadata[\"citations\"] = 41\n", "model.metadata[\"citations_date\"] = \"2026-07-05\"\n" ] }, { "cell_type": "markdown", "id": "74492239-5aae-4026-9d90-6bc9c574c110", "metadata": {}, "source": [ "## Download clock dependencies" ] }, { "cell_type": "markdown", "id": "971ff75c-c5bf-4e8e-89d7-01861e9bc107", "metadata": {}, "source": [ "#### Download from R package" ] }, { "cell_type": "code", "execution_count": 5, "id": "5dd71ca3", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Downloading...\n", "From (original): https://drive.google.com/uc?id=14HBIhA-gkp9-RWD0HVAekRGWOHQBqvhc\n", "From (redirected): https://drive.google.com/uc?id=14HBIhA-gkp9-RWD0HVAekRGWOHQBqvhc&confirm=t&uuid=59da83cb-ce32-43c2-829a-c88b3e648a8b\n", "To: /Users/lucascamillo/pyaging/clocks/notebooks/SystemsAge_data.RData\n", "100%|██████████| 3.86G/3.86G [09:18<00:00, 6.92MB/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Downloaded: True -> SystemsAge_data.RData\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "FILE_ID = \"14HBIhA-gkp9-RWD0HVAekRGWOHQBqvhc\"\n", "OUT_PATH = \"SystemsAge_data.RData\"\n", "\n", "if not os.path.exists(OUT_PATH):\n", " try:\n", " import gdown # type: ignore\n", " except Exception:\n", " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"gdown\"])\n", " import gdown # type: ignore\n", " url = f\"https://drive.google.com/uc?id={FILE_ID}\"\n", " gdown.download(url, OUT_PATH, quiet=False)\n", "\n", "print(f\"Downloaded: {os.path.exists(OUT_PATH)} -> {OUT_PATH}\")\n" ] }, { "cell_type": "code", "execution_count": 6, "id": "c2cfbee5-bcaf-46f0-a8d8-3b6150e09bf0", "metadata": { "execution": { "iopub.execute_input": "2024-03-05T21:06:57.822263Z", "iopub.status.busy": "2024-03-05T21:06:57.821863Z", "iopub.status.idle": "2024-03-05T21:06:57.830369Z", "shell.execute_reply": "2024-03-05T21:06:57.829140Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Writing export_systemsage.R\n" ] } ], "source": [ "%%writefile export_systemsage.R\n", "suppressPackageStartupMessages(library(jsonlite))\n", "suppressPackageStartupMessages(library(readr))\n", "suppressPackageStartupMessages(library(tibble))\n", "\n", "use_arrow <- requireNamespace(\"arrow\", quietly = TRUE)\n", "\n", "write_matrix <- function(df, base_name) {\n", " if (use_arrow) {\n", " parquet_path <- paste0(base_name, \".parquet\")\n", " written <- tryCatch({\n", " arrow::write_parquet(df, parquet_path, compression = \"snappy\")\n", " TRUE\n", " }, error = function(e) {\n", " message(sprintf(\"arrow parquet export failed for %s (%s); writing gzip CSV instead.\", base_name, e$message))\n", " FALSE\n", " })\n", " if (isTRUE(written)) {\n", " return(parquet_path)\n", " }\n", " if (file.exists(parquet_path) && file.info(parquet_path)$size == 0) {\n", " unlink(parquet_path)\n", " }\n", " }\n", " csv_path <- paste0(base_name, \".csv.gz\")\n", " readr::write_csv(df, csv_path)\n", " return(csv_path)\n", "}\n", "\n", "rds <- \"SystemsAge_data.RData\"\n", "if (!file.exists(rds)) {\n", " stop(paste(\"Missing\", rds, \"- run the download cell first.\"))\n", "}\n", "load(rds)\n", "\n", "# 1) CpG list used by the clock\n", "CpGs <- rownames(DNAmPCA$rotation)\n", "write_json(CpGs, \"systemsage_CpGs.json\", auto_unbox = TRUE)\n", "\n", "# 2) Mean-imputation values for missing CpGs (named vector)\n", "write_json(as.list(imputeMissingCpGs), \"systemsage_impute_means.json\", digits = 10, auto_unbox = TRUE)\n", "\n", "# 3) DNAm PCA components: rotation matrix plus supporting statistics\n", "rot_tbl <- as_tibble(DNAmPCA$rotation, rownames = \"CpG\")\n", "rotation_path <- write_matrix(rot_tbl, \"systemsage_dnam_pca_rotation\")\n", "\n", "write_json(as.list(DNAmPCA$center), \"systemsage_dnam_pca_center.json\", digits = 10, auto_unbox = TRUE)\n", "write_json(DNAmPCA$scale, \"systemsage_dnam_pca_scale.json\", digits = 10, auto_unbox = TRUE)\n", "write_json(as.numeric(DNAmPCA$sdev), \"systemsage_dnam_pca_sdev.json\", digits = 10, auto_unbox = TRUE)\n", "\n", "# 4) Map DNAm PCs to system-level components\n", "svc_tbl <- as_tibble(system_vector_coefficients, rownames = \"PC\")\n", "system_vector_path <- write_matrix(svc_tbl, \"systemsage_system_vector_coefficients\")\n", "\n", "write_json(as.list(system_scores_coefficients_scale), \"systemsage_system_scores_coefficients_scale.json\", digits = 10, auto_unbox = TRUE)\n", "\n", "component_map <- tibble(\n", " component = colnames(system_vector_coefficients),\n", " system = gsub(\"[0-9]+$\", \"\", component)\n", ")\n", "write_csv(component_map, \"systemsage_system_component_map.csv\")\n", "\n", "# 5) Predicted chronological age model (DNAm PCs -> age before calibration)\n", "pred_age <- list(\n", " intercept = as.numeric(Predicted_age_coefficients[1]),\n", " loadings = as.numeric(Predicted_age_coefficients[-1])\n", ")\n", "write_json(pred_age, \"systemsage_predicted_age_linear.json\", digits = 10, auto_unbox = TRUE)\n", "write_json(list(a = Age_prediction_model[1], b = Age_prediction_model[2], c = Age_prediction_model[3]),\n", " \"systemsage_predicted_age_poly.json\", digits = 10, auto_unbox = TRUE)\n", "\n", "# 6) PCA on system scores (11 systems + age prediction)\n", "sys_rot_tbl <- as_tibble(systems_PCA$rotation, rownames = \"variable\")\n", "systems_pca_rotation_path <- write_matrix(sys_rot_tbl, \"systemsage_systems_pca_rotation\")\n", "write_json(as.list(systems_PCA$center), \"systemsage_systems_pca_center.json\", digits = 10, auto_unbox = TRUE)\n", "write_json(as.list(systems_PCA$scale), \"systemsage_systems_pca_scale.json\", digits = 10, auto_unbox = TRUE)\n", "write_json(as.numeric(systems_PCA$sdev), \"systemsage_systems_pca_sdev.json\", digits = 10, auto_unbox = TRUE)\n", "\n", "# 7) Final SystemsAge coefficients (systems PCA -> SystemsAge)\n", "write_json(as.numeric(Systems_clock_coefficients), \"systemsage_final_coefficients.json\", digits = 10, auto_unbox = TRUE)\n", "\n", "# 8) Transformation coefficients for scaling outputs (11 systems + age prediction + SystemsAge)\n", "system_labels <- c(\"Blood\", \"Brain\", \"Inflammation\", \"Heart\", \"Hormone\", \"Immune\", \"Kidney\", \"Liver\", \"Metabolic\", \"Lung\", \"MusculoSkeletal\", \"Age_prediction\", \"SystemsAge\")\n", "trans_tbl <- tibble(\n", " label = system_labels,\n", " Coef1 = transformation_coefs[, 1],\n", " Coef2 = transformation_coefs[, 2],\n", " Coef3 = transformation_coefs[, 3],\n", " Coef4 = transformation_coefs[, 4]\n", ")\n", "write_csv(trans_tbl, \"systemsage_transformation_coeffs.csv\")\n", "\n", "# 9) Metadata summary\n", "meta <- list(\n", " n_cpgs = length(CpGs),\n", " n_dnam_pcs = ncol(DNAmPCA$rotation),\n", " n_system_components = ncol(system_vector_coefficients),\n", " n_systems = length(system_labels),\n", " files = list(\n", " dnam_pca_rotation = rotation_path,\n", " system_vector_coefficients = system_vector_path,\n", " systems_pca_rotation = systems_pca_rotation_path\n", " )\n", ")\n", "write_json(meta, \"systemsage_export_summary.json\", auto_unbox = TRUE)\n", "\n", "cat(\"Export complete\\n\")\n", "\n" ] }, { "cell_type": "code", "execution_count": 7, "id": "b71a6f03-61ba-462f-a2a6-a5df95e105ff", "metadata": { "execution": { "iopub.execute_input": "2024-03-05T21:06:57.834701Z", "iopub.status.busy": "2024-03-05T21:06:57.834437Z", "iopub.status.idle": "2024-03-05T21:07:18.968803Z", "shell.execute_reply": "2024-03-05T21:07:18.968475Z" } }, "outputs": [ { "data": { "text/plain": [ "0" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "os.system(\"Rscript export_systemsage.R\")" ] }, { "cell_type": "markdown", "id": "a14c7fc1-abe5-42a3-8bc9-0987521ddf33", "metadata": {}, "source": [ "## Load features" ] }, { "cell_type": "markdown", "id": "0db234ba", "metadata": {}, "source": [ "#### From exported assets\n" ] }, { "cell_type": "code", "execution_count": 8, "id": "97e5b47b-0599-4ec3-aab4-dcfe9d3e4515", "metadata": { "execution": { "iopub.execute_input": "2024-03-05T21:07:18.970732Z", "iopub.status.busy": "2024-03-05T21:07:18.970631Z", "iopub.status.idle": "2024-03-05T21:07:18.975473Z", "shell.execute_reply": "2024-03-05T21:07:18.975187Z" } }, "outputs": [], "source": [ "with open('systemsage_CpGs.json', 'r') as f:\n", " model.features = json.load(f)" ] }, { "cell_type": "markdown", "id": "ee6d8fa0-4767-4c45-9717-eb1c95e2ddc0", "metadata": {}, "source": [ "## Load weights into base model" ] }, { "cell_type": "markdown", "id": "de92ee28-39b1-4356-a734-6b28a20e7bfe", "metadata": {}, "source": [ "#### From JSON file" ] }, { "cell_type": "code", "execution_count": 9, "id": "e09b3463-4fd4-41b1-ac21-e63ddd223fe0", "metadata": { "execution": { "iopub.execute_input": "2024-03-05T21:07:18.977073Z", "iopub.status.busy": "2024-03-05T21:07:18.976993Z", "iopub.status.idle": "2024-03-05T21:07:20.454325Z", "shell.execute_reply": "2024-03-05T21:07:20.454033Z" } }, "outputs": [ { "data": { "text/plain": [ "{'n_cpgs': 125175,\n", " 'n_dnam_pcs': 4018,\n", " 'n_system_components': 84,\n", " 'n_systems': 13,\n", " 'files': {'dnam_pca_rotation': 'systemsage_dnam_pca_rotation.csv.gz',\n", " 'system_vector_coefficients': 'systemsage_system_vector_coefficients.csv.gz',\n", " 'systems_pca_rotation': 'systemsage_systems_pca_rotation.csv.gz'}}" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "with open('systemsage_predicted_age_linear.json', 'r') as f:\n", " predicted_age_linear = json.load(f)\n", "\n", "with open('systemsage_predicted_age_poly.json', 'r') as f:\n", " predicted_age_poly = json.load(f)\n", "\n", "with open('systemsage_system_scores_coefficients_scale.json', 'r') as f:\n", " system_scores_coeff_scale = json.load(f)\n", "\n", "with open('systemsage_systems_pca_center.json', 'r') as f:\n", " systems_pca_center = json.load(f)\n", "\n", "with open('systemsage_systems_pca_scale.json', 'r') as f:\n", " systems_pca_scale = json.load(f)\n", "\n", "with open('systemsage_dnam_pca_center.json', 'r') as f:\n", " dnam_pca_center = json.load(f)\n", "\n", "with open('systemsage_dnam_pca_scale.json', 'r') as f:\n", " dnam_pca_scale = json.load(f)\n", "\n", "with open('systemsage_final_coefficients.json', 'r') as f:\n", " systemsage_final_coeffs = json.load(f)\n", "\n", "transformation_coeffs = pd.read_csv('systemsage_transformation_coeffs.csv')\n", "system_component_map = pd.read_csv('systemsage_system_component_map.csv')\n", "systems_pca_asset = Path('systemsage_systems_pca_rotation.csv.gz')\n", "rotation_asset = Path('systemsage_dnam_pca_rotation.csv.gz')\n", "system_vector_asset = Path('systemsage_system_vector_coefficients.csv.gz')\n", "\n", "with open('systemsage_export_summary.json', 'r') as f:\n", " export_summary = json.load(f)\n", "\n", "export_summary\n" ] }, { "cell_type": "markdown", "id": "c1aa2b7c", "metadata": {}, "source": [ "#### Build SystemsAge base model\n" ] }, { "cell_type": "code", "execution_count": 10, "id": "00857592", "metadata": {}, "outputs": [], "source": [ "# DNAm PCA rotation and centering\n", "rotation_df = pd.read_csv(rotation_asset, compression='gzip')\n", "if rotation_df['CpG'].tolist() != model.features:\n", " raise ValueError('Feature order mismatch between rotation matrix and model features')\n", "model.dnam_rotation = torch.tensor(rotation_df.drop(columns=['CpG']).to_numpy(dtype=np.float32))\n", "model.dnam_center = torch.tensor([dnam_pca_center[cpg] for cpg in model.features], dtype=torch.float32)\n", "del rotation_df\n", "\n", "# System vector coefficients (DNAm PCs -> system components)\n", "system_vector_df = pd.read_csv(system_vector_asset, compression='gzip')\n", "component_names = system_vector_df.columns.tolist()[1:]\n", "model.system_vector = torch.tensor(\n", " system_vector_df.drop(columns=['PC']).fillna(0.0).to_numpy(dtype=np.float32)\n", ")\n", "del system_vector_df\n", "component_index = {name: idx for idx, name in enumerate(component_names)}\n", "\n", "system_label_map = {'Inflammation': 'Cytokine'}\n", "system_labels = transformation_coeffs['label'].tolist()[:11]\n", "model.system_labels = system_labels\n", "\n", "system_indices = []\n", "system_modules = []\n", "for label in system_labels:\n", " lookup = system_label_map.get(label, label)\n", " components = system_component_map.loc[\n", " system_component_map['system'] == lookup, 'component'\n", " ].tolist()\n", " if not components:\n", " raise ValueError(f'No components found for system {label}')\n", "\n", " indices = torch.tensor([component_index[c] for c in components], dtype=torch.long)\n", " weights = torch.tensor([system_scores_coeff_scale[c] for c in components], dtype=torch.float32)\n", "\n", " if weights.numel() == 1:\n", " weights = torch.tensor([-1.0], dtype=torch.float32)\n", "\n", " linear = pya.models.LinearModel(weights.numel())\n", " linear.linear.weight.data.zero_()\n", " linear.linear.bias.data.zero_()\n", " linear.linear.weight.data.copy_(weights.view(1, -1))\n", " linear.linear.bias.requires_grad_(False)\n", " linear.linear.weight.requires_grad_(False)\n", "\n", " system_indices.append(indices)\n", " system_modules.append(linear)\n", "\n", "model.system_component_indices = system_indices\n", "model.system_modules = nn.ModuleList(system_modules)\n", "\n", "# Predicted chronological age assets\n", "predicted_age_weights = torch.tensor(predicted_age_linear['loadings'], dtype=torch.float32)\n", "predicted_age_model = pya.models.LinearModel(predicted_age_weights.numel())\n", "predicted_age_model.linear.weight.data.zero_()\n", "predicted_age_model.linear.bias.data.zero_()\n", "predicted_age_model.linear.weight.data.copy_(predicted_age_weights.unsqueeze(0))\n", "predicted_age_model.linear.bias.data.fill_(float(predicted_age_linear['intercept']))\n", "for param in predicted_age_model.parameters():\n", " param.requires_grad = False\n", "model.predicted_age_model = predicted_age_model\n", "model.predicted_age_poly = torch.tensor(\n", " [predicted_age_poly['a'], predicted_age_poly['b'], predicted_age_poly['c']],\n", " dtype=torch.float32,\n", ")\n", "\n", "# Systems PCA assets (system scores + age prediction)\n", "systems_pca_order = transformation_coeffs['label'].tolist()[:12]\n", "systems_pca_df = pd.read_csv(systems_pca_asset, compression='gzip').set_index('variable')\n", "rotation_cols = [col for col in systems_pca_df.columns if col.startswith('PC')]\n", "scale_vec = torch.tensor([systems_pca_scale[label] for label in systems_pca_order], dtype=torch.float32)\n", "center_vec = torch.tensor([systems_pca_center[label] for label in systems_pca_order], dtype=torch.float32)\n", "rotation_mat = torch.tensor(\n", " systems_pca_df.loc[systems_pca_order, rotation_cols].to_numpy(dtype=np.float32)\n", ")\n", "rotation_scaled = rotation_mat / scale_vec.unsqueeze(1)\n", "\n", "systems_pca_model = pya.models.PCLinearModel(input_dim=len(systems_pca_order), pc_dim=rotation_scaled.size(1))\n", "systems_pca_model.center.data.copy_(center_vec)\n", "systems_pca_model.rotation.data.copy_(rotation_scaled)\n", "systems_pca_model.linear.weight.data.zero_()\n", "systems_pca_model.linear.bias.data.zero_()\n", "systems_pca_model.linear.weight.data.copy_(\n", " torch.tensor(systemsage_final_coeffs, dtype=torch.float32).view(1, -1)\n", ")\n", "systems_pca_model.linear.bias.requires_grad_(False)\n", "systems_pca_model.linear.weight.requires_grad_(False)\n", "for param in systems_pca_model.parameters():\n", " param.requires_grad = False\n", "model.systems_pca_model = systems_pca_model\n", "\n", "# Transformation assets\n", "model.transformation_coefs = torch.tensor(\n", " transformation_coeffs[['Coef1', 'Coef2', 'Coef3', 'Coef4']].to_numpy(dtype=np.float32)\n", ")\n", "model.transformation_labels = transformation_coeffs['label'].tolist()\n", "\n", "# Record summary metadata for downstream reference\n", "model.export_summary = export_summary" ] }, { "cell_type": "markdown", "id": "93271aee-d045-45ba-b030-7ec8e57add42", "metadata": {}, "source": [ "#### From JSON file" ] }, { "cell_type": "code", "execution_count": 11, "id": "2089b66f-9cc4-4528-9bdc-5e45efc6d06b", "metadata": { "execution": { "iopub.execute_input": "2024-03-05T21:07:20.460308Z", "iopub.status.busy": "2024-03-05T21:07:20.460214Z", "iopub.status.idle": "2024-03-05T21:07:20.467639Z", "shell.execute_reply": "2024-03-05T21:07:20.467424Z" } }, "outputs": [], "source": [ "with open('systemsage_impute_means.json', 'r') as f:\n", " reference_feature_values = json.load(f)\n", "model.reference_values = torch.tensor(\n", " [reference_feature_values[cpg] for cpg in model.features], dtype=torch.float32\n", ")" ] }, { "cell_type": "markdown", "id": "af3bcf7b-74a8-4d21-9ccb-4de0c2b0516b", "metadata": {}, "source": [ "## Load preprocess and postprocess objects" ] }, { "cell_type": "code", "execution_count": 12, "id": "7a22fb20-c605-424d-8efb-7620c2c0755c", "metadata": { "execution": { "iopub.execute_input": "2024-03-05T21:07:20.469160Z", "iopub.status.busy": "2024-03-05T21:07:20.469087Z", "iopub.status.idle": "2024-03-05T21:07:20.470669Z", "shell.execute_reply": "2024-03-05T21:07:20.470435Z" } }, "outputs": [], "source": [ "model.preprocess_name = None\n", "model.preprocess_dependencies = None" ] }, { "cell_type": "code", "execution_count": 13, "id": "ff4a21cb-cf41-44dc-9ed1-95cf8aa15772", "metadata": { "execution": { "iopub.execute_input": "2024-03-05T21:07:20.472006Z", "iopub.status.busy": "2024-03-05T21:07:20.471932Z", "iopub.status.idle": "2024-03-05T21:07:20.473523Z", "shell.execute_reply": "2024-03-05T21:07:20.473253Z" } }, "outputs": [], "source": [ "model.postprocess_name = None\n", "model.postprocess_dependencies = None\n" ] }, { "cell_type": "code", "execution_count": 14, "id": "2168355c-47d9-475d-b816-49f65e74887c", "metadata": { "execution": { "iopub.execute_input": "2024-03-05T21:07:20.474959Z", "iopub.status.busy": "2024-03-05T21:07:20.474880Z", "iopub.status.idle": "2024-03-05T21:07:20.477725Z", "shell.execute_reply": "2024-03-05T21:07:20.477458Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "%==================================== Model Details ====================================%\n", "Model Attributes:\n", "\n", "training: True\n", "metadata: {'approved_by_author': '⌛',\n", " 'citation': 'Sehgal, Raghav, et al. \"Systems Age: A single blood methylation '\n", " 'test to quantify aging heterogeneity across 11 physiological '\n", " 'systems.\" Nature Aging (2025): 1-17.',\n", " 'clock_name': 'systemsage',\n", " 'data_type': 'methylation',\n", " 'doi': 'https://doi.org/10.1038/s43587-025-00958-3',\n", " 'notes': None,\n", " 'research_only': True,\n", " 'species': 'Homo sapiens',\n", " 'version': None,\n", " 'year': 2025}\n", "reference_values: [0.7703027725219727, 0.10787541419267654, 0.14593413472175598, 0.15467368066310883, 0.05177399888634682, 0.7689204812049866, 0.09734559804201126, 0.5384307503700256, 0.03857552260160446, 0.14853332936763763, 0.03710409998893738, 0.7541770339012146, 0.05995236337184906, 0.7082317471504211, 0.9218639731407166, 0.04736267030239105, 0.11038260161876678, 0.16661499440670013, 0.08505702018737793, 0.8980633020401001, 0.44178515672683716, 0.04835095629096031, 0.2284272015094757, 0.04166311398148537, 0.04188517853617668, 0.1550152450799942, 0.18579867482185364, 0.14117033779621124, 0.04062004014849663, 0.046620775014162064]... [Tensor of shape torch.Size([125175])]\n", "preprocess_name: None\n", "preprocess_dependencies: None\n", "postprocess_name: None\n", "postprocess_dependencies: None\n", "features: ['cg01763666', 'cg02115394', 'cg19607165', 'cg15998406', 'cg11890956', 'cg11947782', 'cg14361409', 'cg00376553', 'cg21650422', 'cg22782271', 'cg00172270', 'cg17505155', 'cg14428027', 'cg25593794', 'cg26676405', 'cg15241920', 'cg25857090', 'cg00347904', 'cg18764577', 'cg01846046', 'cg04586126', 'cg15004949', 'cg24500959', 'cg15283028', 'cg08193650', 'cg13481974', 'cg21855135', 'cg14491479', 'cg11739541', 'cg26196700']... [Total elements: 125175]\n", "base_model_features: None\n", "base_model: None\n", "prediction_index: -1\n", "dnam_center: [0.8258556723594666, 0.05573316663503647, 0.058175552636384964, 0.16283440589904785, 0.020828289911150932, 0.7620843648910522, 0.08140864223241806, 0.4882557988166809, 0.019841225817799568, 0.05279502272605896, 0.01341407373547554, 0.8174307942390442, 0.024278903380036354, 0.6846079230308533, 0.9660204648971558, 0.02498844638466835, 0.0645182803273201, 0.06545444577932358, 0.05297725275158882, 0.9388992786407471, 0.33690470457077026, 0.0214790478348732, 0.15520553290843964, 0.014546176418662071, 0.014784707687795162, 0.15396034717559814, 0.1592680811882019, 0.05003071576356888, 0.012521471828222275, 0.017569566145539284]... [Tensor of shape torch.Size([125175])]\n", "dnam_rotation: [-0.0006869738572277129, 0.0038640284910798073, -0.0012635828461498022, 0.0007870842237025499, -0.0012677146587520838, 0.006102594546973705, 0.006675826385617256, -0.0004881693748757243, 0.0031523199286311865, -0.002742027398198843, 0.004634317941963673, -0.0022342982701957226, -0.003684432478621602, 0.005017344374209642, -0.00214009010232985, -0.004212546162307262, -0.001904419157654047, -0.001993611454963684, 0.0002256415318697691, 0.004155374597758055, 0.0018408787436783314, -0.004885646980255842, 0.00027931606746278703, 0.0003587851533666253, -0.0005096819368191063, -0.00012601316848304123, -0.0004541595990303904, -0.0010596434585750103, -0.0007061095675453544, -0.0007520628860220313]... [Tensor of shape torch.Size([125175, 4018])]\n", "system_vector: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0002808159915730357, 0.0, 0.0, 0.27329912781715393, 0.0, 0.0, -0.11009273678064346, 0.19336724281311035, -0.14641909301280975, -0.761561393737793, 2.403021812438965, 0.15886220335960388, -0.03239140659570694, -0.07976681739091873, -0.06844867765903473, 0.0, -0.0005787260015495121]... [Tensor of shape torch.Size([4018, 84])]\n", "system_labels: ['Blood',\n", " 'Brain',\n", " 'Inflammation',\n", " 'Heart',\n", " 'Hormone',\n", " 'Immune',\n", " 'Kidney',\n", " 'Liver',\n", " 'Metabolic',\n", " 'Lung',\n", " 'MusculoSkeletal']\n", "system_component_indices: [tensor([0, 1, 2, 3, 4, 5, 6]),\n", " tensor([ 7, 8, 9, 10, 11, 12]),\n", " tensor([13, 14, 15]),\n", " tensor([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]),\n", " tensor([28]),\n", " tensor([29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46,\n", " 47]),\n", " tensor([48, 49, 50, 51, 52, 53]),\n", " tensor([54, 55, 56, 57, 58, 59]),\n", " tensor([65, 66, 67, 68, 69, 70]),\n", " tensor([60, 61, 62, 63, 64]),\n", " tensor([71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83])]\n", "predicted_age_poly: tensor([-6.4341e+02, 2.1224e+00, -4.3686e-04])\n", "transformation_coefs: [-0.13082726299762726, 0.08089754730463028, 701.9954223632812, 158.3132781982422, -0.20473948121070862, 0.10236957669258118, 701.9954223632812, 158.3132781982422, -0.050485897809267044, 0.12262580543756485, 701.9954223632812, 158.3132781982422, -0.1948918253183365, 0.14121989905834198, 701.9954223632812, 158.3132781982422, -0.3021743595600128, 0.24332834780216217, 701.9954223632812, 158.3132781982422, 0.04260144382715225, 0.041371069848537445, 701.9954223632812, 158.3132781982422, -0.10534755140542984, 0.17005649209022522, 701.9954223632812, 158.3132781982422, -0.08745717257261276, 0.18173155188560486]... [Tensor of shape torch.Size([13, 4])]\n", "transformation_labels: ['Blood',\n", " 'Brain',\n", " 'Inflammation',\n", " 'Heart',\n", " 'Hormone',\n", " 'Immune',\n", " 'Kidney',\n", " 'Liver',\n", " 'Metabolic',\n", " 'Lung',\n", " 'MusculoSkeletal',\n", " 'Age_prediction',\n", " 'SystemsAge']\n", "export_summary: {'files': {'dnam_pca_rotation': 'systemsage_dnam_pca_rotation.csv.gz',\n", " 'system_vector_coefficients': 'systemsage_system_vector_coefficients.csv.gz',\n", " 'systems_pca_rotation': 'systemsage_systems_pca_rotation.csv.gz'},\n", " 'n_cpgs': 125175,\n", " 'n_dnam_pcs': 4018,\n", " 'n_system_components': 84,\n", " 'n_systems': 13}\n", "\n", "%==================================== Model Details ====================================%\n", "Model Structure:\n", "\n", "system_modules: ModuleList(\n", " (0): LinearModel(\n", " (linear): Linear(in_features=7, out_features=1, bias=True)\n", " )\n", " (1): LinearModel(\n", " (linear): Linear(in_features=6, out_features=1, bias=True)\n", " )\n", " (2): LinearModel(\n", " (linear): Linear(in_features=3, out_features=1, bias=True)\n", " )\n", " (3): LinearModel(\n", " (linear): Linear(in_features=12, out_features=1, bias=True)\n", " )\n", " (4): LinearModel(\n", " (linear): Linear(in_features=1, out_features=1, bias=True)\n", " )\n", " (5): LinearModel(\n", " (linear): Linear(in_features=19, out_features=1, bias=True)\n", " )\n", " (6-8): 3 x LinearModel(\n", " (linear): Linear(in_features=6, out_features=1, bias=True)\n", " )\n", " (9): LinearModel(\n", " (linear): Linear(in_features=5, out_features=1, bias=True)\n", " )\n", " (10): LinearModel(\n", " (linear): Linear(in_features=13, out_features=1, bias=True)\n", " )\n", ")\n", "predicted_age_model: LinearModel(\n", " (linear): Linear(in_features=4018, out_features=1, bias=True)\n", ")\n", "systems_pca_model: PCLinearModel(\n", " (linear): Linear(in_features=12, out_features=1, bias=True)\n", ")\n", "\n", "%==================================== Model Details ====================================%\n", "Model Parameters and Weights:\n", "\n", "system_modules.0.linear.weight: tensor([[ 0.2417, -0.0264, 0.0000, -0.0299, -0.1483, 0.0530, -0.2911]])\n", "system_modules.0.linear.bias: tensor([0.])\n", "system_modules.1.linear.weight: tensor([[ 0.4734, 0.0093, -0.3984, -0.2834, 0.0000, 0.0000]])\n", "system_modules.1.linear.bias: tensor([0.])\n", "system_modules.2.linear.weight: tensor([[ 0.2195, 0.2346, -0.1890]])\n", "system_modules.2.linear.bias: tensor([0.])\n", "system_modules.3.linear.weight: tensor([[-0.0019, -0.0073, 0.0006, -0.0120, 0.0020, 0.0052, 0.0023, 0.0004,\n", " 0.0032, -0.0007, -0.0191, -0.0033]])\n", "system_modules.3.linear.bias: tensor([0.])\n", "system_modules.4.linear.weight: tensor([[-1.]])\n", "system_modules.4.linear.bias: tensor([0.])\n", "system_modules.5.linear.weight: tensor([[-0.0101, 0.0143, 0.0000, 0.0023, 0.0455, -0.0198, -0.0396, 0.0034,\n", " 0.0000, 0.0000, -0.0945, -0.1349, 0.0063, 0.1958, 0.0000, -0.0651,\n", " -0.0975, -0.2673, 0.6696]])\n", "system_modules.5.linear.bias: tensor([0.])\n", "system_modules.6.linear.weight: tensor([[ 0.2375, 0.2883, 0.0232, -0.3228, 0.6635, 0.2668]])\n", "system_modules.6.linear.bias: tensor([0.])\n", "system_modules.7.linear.weight: tensor([[ 0.2323, -0.0565, 0.2820, 1.3307, 1.1750, -0.5682]])\n", "system_modules.7.linear.bias: tensor([0.])\n", "system_modules.8.linear.weight: tensor([[ 0.1536, 0.0000, 0.4502, -0.3308, -0.0246, -0.6582]])\n", "system_modules.8.linear.bias: tensor([0.])\n", "system_modules.9.linear.weight: tensor([[-0.0270, -0.2845, 0.2487, 0.0478, 0.2502]])\n", "system_modules.9.linear.bias: tensor([0.])\n", "system_modules.10.linear.weight: tensor([[-0.0066, -0.0189, -0.0063, -0.0091, 0.0029, 0.0033, 0.0000, -0.0229,\n", " -0.0009, -0.0169, 0.0148, 0.0308, 0.0167]])\n", "system_modules.10.linear.bias: tensor([0.])\n", "predicted_age_model.linear.weight: [-0.2724407911300659, 1.3232460021972656, 0.0, 11.594740867614746, 2.8045454025268555, 10.507875442504883, 0.38123732805252075, 0.0, 24.096769332885742, 4.509000778198242, 7.230065822601318, 17.05602264404297, -17.76302146911621, -12.903092384338379, -10.236844062805176, -8.605022430419922, 2.662332534790039, 0.0, -0.39519476890563965, 11.368020057678223, -3.425773859024048, -4.283892631530762, 14.685194969177246, -5.893307209014893, 3.771191358566284, -25.85748863220215, 0.0, -2.187864065170288, -2.016566514968872, 0.0]... [Tensor of shape torch.Size([1, 4018])]\n", "predicted_age_model.linear.bias: tensor([855.7034])\n", "systems_pca_model.center: tensor([-1.3083e-01, -2.0474e-01, -5.0486e-02, -1.9489e-01, -1.0903e-01,\n", " 4.2601e-02, -1.0535e-01, -8.7457e-02, -2.1566e-01, 9.7069e-02,\n", " 1.0829e-02, 6.0997e+01])\n", "systems_pca_model.rotation: [1.609099268913269, -0.06755393743515015, -1.0445926189422607, 0.33840256929397583, -1.9766627550125122, 3.0764424800872803, -2.3690080642700195, 0.1746765673160553, -1.2944000959396362, -0.45856714248657227, 1.5674749612808228, 0.6277533769607544, 1.1023108959197998, 1.7608975172042847, -1.2658442258834839, 0.8253107070922852, -1.4267181158065796, -0.2949528396129608, 0.23251882195472717, -1.6551636457443237, 1.9466115236282349, 0.8668460249900818, -0.7407866716384888, -0.3994610905647278, 1.125208854675293, -0.009071183390915394, 0.16698575019836426, -0.5564174056053162, 0.650899350643158, -0.5170385837554932]... [Tensor of shape torch.Size([12, 12])]\n", "systems_pca_model.linear.weight: tensor([[ 1.2396, 0.6278, 2.1838, -0.1105, -0.6525, -0.4498, -0.9124, 0.4811,\n", " 0.0000, 0.2999, 0.0464, -0.0344]])\n", "systems_pca_model.linear.bias: tensor([0.])\n", "\n", "%==================================== Model Details ====================================%\n", "\n" ] } ], "source": [ "pya.utils.print_model_details(model)" ] }, { "cell_type": "markdown", "id": "986d0262-e0c7-4036-b687-dee53ba392fb", "metadata": {}, "source": [ "## Basic test" ] }, { "cell_type": "code", "execution_count": 15, "id": "936b9877-d076-4ced-99aa-e8d4c58c5caf", "metadata": { "execution": { "iopub.execute_input": "2024-03-05T21:07:20.479197Z", "iopub.status.busy": "2024-03-05T21:07:20.479121Z", "iopub.status.idle": "2024-03-05T21:07:20.515391Z", "shell.execute_reply": "2024-03-05T21:07:20.515059Z" } }, "outputs": [ { "data": { "text/plain": [ "tensor([[-13.8881]])" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "torch.manual_seed(42)\n", "input_sample = torch.randn(1, len(model.features), dtype=torch.float32)\n", "model.eval()\n", "model.to(torch.float32)\n", "pred = model(input_sample)\n", "pred\n" ] }, { "cell_type": "markdown", "id": "fe8299d7-9285-4e22-82fd-b664434b4369", "metadata": {}, "source": [ "## Save torch model" ] }, { "cell_type": "code", "execution_count": 16, "id": "5ef2fa8d-c80b-4fdd-8555-79c0d541788e", "metadata": { "execution": { "iopub.execute_input": "2024-03-05T21:07:20.517358Z", "iopub.status.busy": "2024-03-05T21:07:20.517244Z", "iopub.status.idle": "2024-03-05T21:07:20.668047Z", "shell.execute_reply": "2024-03-05T21:07:20.667536Z" } }, "outputs": [], "source": [ "torch.save(model, f\"../weights/{model.metadata['clock_name']}.pt\")" ] }, { "cell_type": "markdown", "id": "bac6257b-8d08-4a90-8d0b-7f745dc11ac1", "metadata": {}, "source": [ "## Clear directory\n", "" ] }, { "cell_type": "code", "execution_count": 17, "id": "11aeaa70-44c0-42f9-86d7-740e3849a7a6", "metadata": { "execution": { "iopub.execute_input": "2024-03-05T21:07:20.670161Z", "iopub.status.busy": "2024-03-05T21:07:20.670044Z", "iopub.status.idle": "2024-03-05T21:07:20.775559Z", "shell.execute_reply": "2024-03-05T21:07:20.775255Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Deleted file: systemsage_transformation_coeffs.csv\n", "Deleted file: systemsage_impute_means.json\n", "Deleted file: systemsage_export_summary.json\n", "Deleted file: systemsage_systems_pca_scale.json\n", "Deleted folder: methylCIPHER\n", "Deleted file: systemsage_dnam_pca_center.json\n", "Deleted file: systemsage_systems_pca_center.json\n", "Deleted file: export_systemsage.R\n", "Deleted file: systemsage_dnam_pca_rotation.csv.gz\n", "Deleted file: systemsage_system_component_map.csv\n", "Deleted file: systemsage_predicted_age_linear.json\n", "Deleted file: SystemsAge_data.RData\n", "Deleted file: systemsage_predicted_age_poly.json\n", "Deleted file: systemsage_dnam_pca_scale.json\n", "Deleted file: systemsage_systems_pca_rotation.csv.gz\n", "Deleted file: systemsage_CpGs.json\n", "Deleted file: systemsage_dnam_pca_sdev.json\n", "Deleted file: systemsage_final_coefficients.json\n", "Deleted file: systemsage_system_scores_coefficients_scale.json\n", "Deleted file: systemsage_systems_pca_sdev.json\n", "Deleted file: systemsage_system_vector_coefficients.csv.gz\n" ] } ], "source": [ "# Function to remove a folder and all its contents\n", "def remove_folder(path):\n", " try:\n", " shutil.rmtree(path)\n", " print(f\"Deleted folder: {path}\")\n", " except Exception as e:\n", " print(f\"Error deleting folder {path}: {e}\")\n", "\n", "# Get a list of all files and folders in the current directory\n", "all_items = os.listdir('.')\n", "\n", "# Loop through the items\n", "for item in all_items:\n", " # Check if it's a file and does not end with .ipynb\n", " if os.path.isfile(item) and not item.endswith('.ipynb'):\n", " os.remove(item)\n", " print(f\"Deleted file: {item}\")\n", " # Check if it's a folder\n", " elif os.path.isdir(item):\n", " remove_folder(item)" ] } ], "metadata": { "kernelspec": { "display_name": "research", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.7" } }, "nbformat": 4, "nbformat_minor": 5 }