Merge branch 'master' into add-nallo-loqusdb-upload

Author: peterpru

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 69.3.0
+current_version = 69.5.3
 commit = True
 tag = True
 tag_name = v{new_version}

alembic/versions/2025_04_17_6362cfd4c61f_add_nallo_ordertype.py

@@ -0,0 +1,58 @@
+"""add Nallo ordertype
+
+Revision ID: 6362cfd4c61f
+Revises: 3a0250e5526d
+Create Date: 2025-04-17 10:42:20.943466
+
+"""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "6362cfd4c61f"
+down_revision = "3a0250e5526d"
+branch_labels = None
+depends_on = None
+
+old_order_types = [
+    "BALSAMIC",
+    "BALSAMIC_UMI",
+    "FASTQ",
+    "FLUFFY",
+    "METAGENOME",
+    "MICROBIAL_FASTQ",
+    "MICROSALT",
+    "MIP_DNA",
+    "MIP_RNA",
+    "PACBIO_LONG_READ",
+    "RML",
+    "RNAFUSION",
+    "SARS_COV_2",
+    "TAXPROFILER",
+    "TOMTE",
+]
+
+new_order_types = old_order_types.copy()
+new_order_types.append("NALLO")
+
+
+def upgrade():
+    op.alter_column(
+        table_name="order_type_application",
+        column_name="order_type",
+        existing_type=sa.Enum(*old_order_types),
+        type_=sa.Enum(*new_order_types),
+        nullable=False,
+    )
+
+
+def downgrade():
+    op.alter_column(
+        table_name="order_type_application",
+        column_name="order_type",
+        existing_type=sa.Enum(*new_order_types),
+        type_=sa.Enum(*old_order_types),
+        nullable=False,
+    )

cg/__init__.py

@@ -1,2 +1,2 @@
 __title__ = "cg"
-__version__ = "69.3.0"
+__version__ = "69.5.3"

cg/apps/scout/scoutapi.py

@@ -20,9 +20,9 @@
 class ScoutAPI:
     """Interface to Scout."""
 
-    def __init__(self, config, slurm_upload_service: SlurmUploadService):
-        binary_path = config["scout"]["binary_path"]
-        config_path = config["scout"]["config_path"]
+    def __init__(self, scout_config, slurm_upload_service: SlurmUploadService):
+        binary_path = scout_config.binary_path
+        config_path = scout_config.config_path
         self.process = Process(binary=binary_path, config=config_path)
         self.slurm_upload_service = slurm_upload_service
         self.scout_base_command = f"{binary_path} --config {config_path}"

cg/cli/clean.py

@@ -122,7 +122,7 @@ def scout_finished_cases(
     context: click.Context, days_old: int, skip_confirmation: bool = False, dry_run: bool = False
 ) -> None:
     """Clean up of solved and archived Scout cases."""
-    scout_api: ScoutAPI = context.obj.scout_api
+    scout_api: ScoutAPI = context.obj.scout_api_37
     bundles: list[str] = []
     for status in [ScoutTag.ARCHIVED, ScoutTag.SOLVED]:
         cases: list[ScoutExportCase] = scout_api.get_cases(status=status, reruns=False)

cg/cli/deliver/base.py

@@ -15,9 +15,7 @@
 from cg.services.deliver_files.deliver_files_service.deliver_files_service import (
     DeliverFilesService,
 )
-from cg.services.deliver_files.factory import (
-    DeliveryServiceFactory,
-)
+from cg.services.deliver_files.factory import DeliveryServiceFactory
 from cg.services.deliver_files.rsync.service import DeliveryRsyncService
 from cg.store.models import Analysis, Case
 
@@ -29,6 +27,7 @@
     multiple=False,
     type=click.Choice(choices=[option for option in FileDeliveryOption]),
     required=False,
+    help="The delivery type to use. Overrides any delivery type from the case",
 )
 TICKET_ID_ARG = click.option("-t", "--ticket", type=str, required=True)
 

cg/cli/upload/mutacc.py

@@ -7,6 +7,7 @@
 from cg.apps.mutacc_auto import MutaccAutoAPI
 from cg.apps.scout.scout_export import ScoutExportCase
 from cg.apps.scout.scoutapi import ScoutAPI
+from cg.cli.upload.utils import get_scout_api
 from cg.constants.cli_options import DRY_RUN
 from cg.meta.upload.mutacc import UploadToMutaccAPI
 from cg.models.cg_config import CGConfig
@@ -28,7 +29,7 @@ def process_solved(
 
     LOG.info("----------------- PROCESS-SOLVED ----------------")
 
-    scout_api: ScoutAPI = context.scout_api
+    scout_api: ScoutAPI = get_scout_api(cg_config=context, case_id=case_id)
     mutacc_auto_api: MutaccAutoAPI = context.mutacc_auto_api
     mutacc_upload_api = UploadToMutaccAPI(scout_api=scout_api, mutacc_auto_api=mutacc_auto_api)
 

cg/cli/upload/scout.py

@@ -8,7 +8,7 @@
 
 from cg.apps.housekeeper.hk import HousekeeperAPI
 from cg.apps.scout.scoutapi import ScoutAPI
-from cg.cli.upload.utils import suggest_cases_to_upload
+from cg.cli.upload.utils import get_scout_api, suggest_cases_to_upload
 from cg.constants import Workflow
 from cg.constants.cli_options import DRY_RUN
 from cg.constants.constants import FileFormat
@@ -145,7 +145,7 @@ def upload_case_to_scout(context: CGConfig, re_upload: bool, dry_run: bool, case
     LOG.info("----------------- UPLOAD -----------------------")
 
     housekeeper_api: HousekeeperAPI = context.housekeeper_api
-    scout_api: ScoutAPI = context.scout_api
+    scout_api: ScoutAPI = get_scout_api(cg_config=context, case_id=case_id)
 
     tag_name: str = UploadScoutAPI.get_load_config_tag()
     version: Version = housekeeper_api.last_version(bundle=case_id)

cg/cli/upload/utils.py

@@ -4,8 +4,10 @@
 
 import rich_click as click
 
+from cg.apps.scout.scoutapi import ScoutAPI
 from cg.constants import Workflow
 from cg.constants.constants import MAX_ITEMS_TO_RETRIEVE
+from cg.models.cg_config import CGConfig
 from cg.store.models import Analysis
 from cg.store.store import Store
 
@@ -20,3 +22,8 @@ def suggest_cases_to_upload(status_db: Store, workflow: Workflow | None = None)
     ]
     for case_obj in records:
         click.echo(case_obj)
+
+
+def get_scout_api(cg_config: CGConfig, case_id: str) -> ScoutAPI:
+    workflow = cg_config.status_db.get_case_by_internal_id(case_id).data_analysis
+    return cg_config.scout_api_38 if workflow == Workflow.NALLO else cg_config.scout_api_37

cg/constants/constants.py

@@ -176,6 +176,9 @@ class DataDelivery(StrEnum):
     FASTQ_ANALYSIS_SCOUT: str = "fastq-analysis-scout"
     NIPT_VIEWER: str = "nipt-viewer"
     NO_DELIVERY: str = "no-delivery"
+    RAW_DATA_ANALYSIS: str = "raw_data-analysis"
+    RAW_DATA_ANALYSIS_SCOUT: str = "raw_data-analysis-scout"
+    RAW_DATA_SCOUT: str = "raw_data-scout"
     SCOUT: str = "scout"
     STATINA: str = "statina"
 

cg/constants/delivery.py

@@ -68,6 +68,7 @@
 
 RAW_DATA_ANALYSIS_SAMPLE_TAGS: list[set[str]] = [
     {"fastq"},
+    {"bam"},
 ]
 
 MIP_DNA_ANALYSIS_CASE_TAGS: list[set[str]] = [
@@ -200,3 +201,4 @@ class FileDeliveryOption(StrEnum):
     BAM: str = "bam"
     FASTQ: str = "fastq"
     FASTQ_ANALYSIS: str = "fastq-analysis"
+    RAW_DATA_ANALYSIS: str = "raw_data-analysis"

cg/constants/nf_analysis.py

@@ -20,14 +20,11 @@ class NfTowerStatus(StrEnum):
     "predicted_sex_sex_check": {"norm": "eq", "threshold": None},
 }
 
-RAREDISEASE_ADAPTER_BASES_PERCENTAGE_THRESHOLD = 0.005
-
 RAREDISEASE_PREDICTED_SEX_METRIC = "predicted_sex_sex_check"
 
 RAREDISEASE_METRIC_CONDITIONS_WES: dict[str, dict[str, Any]] = {
     "PERCENT_DUPLICATION": {"norm": "lt", "threshold": 0.20},
     "Contamination Status": {"norm": "eq", "threshold": "NO"},
-    "adapter_cutting_adapter_trimmed_reads": {"norm": "lt", "threshold": None},
     "PCT_PF_UQ_READS_ALIGNED": {"norm": "gt", "threshold": 0.95},
     "PCT_TARGET_BASES_10X": {"norm": "gt", "threshold": 0.95},
     "AT_DROPOUT": {"norm": "lt", "threshold": 10},
@@ -39,7 +36,6 @@ class NfTowerStatus(StrEnum):
 RAREDISEASE_METRIC_CONDITIONS_WGS: dict[str, dict[str, Any]] = {
     "PERCENT_DUPLICATION": {"norm": "lt", "threshold": 0.20},
     "Contamination Status": {"norm": "eq", "threshold": "NO"},
-    "adapter_cutting_adapter_trimmed_reads": {"norm": "lt", "threshold": None},
     "PCT_PF_UQ_READS_ALIGNED": {"norm": "gt", "threshold": 0.95},
     "MEDIAN_TARGET_COVERAGE": {"norm": "gt", "threshold": 25},
     "PCT_TARGET_BASES_10X": {"norm": "gt", "threshold": 0.95},

cg/constants/report.py

@@ -37,6 +37,9 @@
     DataDelivery.FASTQ_ANALYSIS_SCOUT,
     DataDelivery.FASTQ_QC_ANALYSIS,
     DataDelivery.FASTQ_SCOUT,
+    DataDelivery.RAW_DATA_ANALYSIS,
+    DataDelivery.RAW_DATA_ANALYSIS_SCOUT,
+    DataDelivery.RAW_DATA_SCOUT,
     DataDelivery.SCOUT,
 )
 

cg/constants/scout.py

@@ -65,6 +65,9 @@ class ScoutUploadKey(StrEnum):
 NALLO_CASE_TAGS = dict(
     delivery_report={"delivery-report"},
     multiqc={"multiqc-html"},
+    peddy_check={"ped-check", "peddy"},
+    peddy_ped={"ped", "peddy"},
+    peddy_sex={"sex-check", "peddy"},
     vcf_snv_research={"vcf-snv-research"},
     vcf_snv={"vcf-snv-clinical"},
     vcf_sv_research={"vcf-sv-research"},
@@ -89,18 +92,6 @@ class ScoutUploadKey(StrEnum):
     vcf_str={"vcf-str"},
 )
 
-NALLO_CASE_TAGS = dict(
-    delivery_report={"delivery-report"},
-    multiqc={"multiqc-html"},
-    somalier_pairs={"relate-pairs", "somalier"},
-    somalier_samples={"relate-samples", "somalier"},
-    vcf_snv_research={"vcf-snv-research"},
-    vcf_snv={"vcf-snv-clinical"},
-    vcf_sv_research={"vcf-sv-research"},
-    vcf_sv={"vcf-sv-clinical"},
-    vcf_str={"vcf-str"},
-)
-
 BALSAMIC_CASE_TAGS = dict(
     sv_vcf={"vcf-sv-clinical"},
     snv_vcf={"vcf-snv-clinical"},
@@ -146,11 +137,14 @@ class ScoutUploadKey(StrEnum):
 )
 
 NALLO_SAMPLE_TAGS: dict[str, set[str]] = dict(
-    alignment_file={"alignment_haplotags"},
+    alignment_path={AlignmentFileTag.BAM, "haplotags"},
+    assembly_alignment_path={AlignmentFileTag.BAM, "assembly"},
     d4_file={"mosdepth_d4"},
+    hificnv_coverage={"hificnv", "bigwig"},
+    paraphase_alignment_path={AlignmentFileTag.BAM, NalloAnalysisTag.PARAPHASE},
+    minor_allele_frequency_wig={"hificnv", "bigwig", "maf"},
 )
 
-
 MIP_SAMPLE_TAGS: dict[str, set[str]] = dict(
     bam_file={"bam"},
     alignment_file={"cram"},
@@ -166,12 +160,6 @@ class ScoutUploadKey(StrEnum):
     mitodel_file={"mitodel"},
 )
 
-NALLO_SAMPLE_TAGS: dict[str, set[str]] = dict(
-    alignment_file={AlignmentFileTag.BAM, NalloAnalysisTag.HAPLOTAGS},
-    d4_file={"d4"},
-    paraphase_alignment_path={AlignmentFileTag.BAM, NalloAnalysisTag.PARAPHASE},
-)
-
 BALSAMIC_SAMPLE_TAGS = dict(
     bam_file={"bam"},
     alignment_file={"cram"},

cg/constants/sequencing.py

@@ -79,5 +79,3 @@ class SeqLibraryPrepCategory(StrEnum):
     SeqLibraryPrepCategory.TARGETED_GENOME_SEQUENCING,
     SeqLibraryPrepCategory.WHOLE_EXOME_SEQUENCING,
 ]
-
-NOVASEQ_SEQUENCING_READ_LENGTH = 151

cg/meta/delivery_report/delivery_report_api.py

@@ -37,14 +37,7 @@
     SampleModel,
     TimestampModel,
 )
-from cg.store.models import (
-    Analysis,
-    Application,
-    ApplicationLimitations,
-    Case,
-    CaseSample,
-    Sample,
-)
+from cg.store.models import Analysis, Application, ApplicationLimitations, Case, CaseSample, Sample
 from cg.store.store import Store
 
 LOG = logging.getLogger(__name__)

cg/meta/delivery_report/mip_dna.py

@@ -19,11 +19,11 @@
 from cg.meta.delivery_report.data_validators import get_million_read_pairs
 from cg.meta.delivery_report.delivery_report_api import DeliveryReportAPI
 from cg.meta.workflow.mip_dna import MipDNAAnalysisAPI
-from cg.models.mip.mip_analysis import MipAnalysis
-from cg.models.mip.mip_metrics_deliverables import get_sample_id_metric
 from cg.models.delivery_report.metadata import MipDNASampleMetadataModel
 from cg.models.delivery_report.report import CaseModel, ReportRequiredFields, ScoutVariantsFiles
 from cg.models.delivery_report.sample import SampleModel
+from cg.models.mip.mip_analysis import MipAnalysis
+from cg.models.mip.mip_metrics_deliverables import get_sample_id_metric
 from cg.store.models import Case, Sample
 
 LOG = logging.getLogger(__name__)

cg/meta/meta.py

@@ -49,6 +49,7 @@ def __init__(self, config: CGConfig):
                 ),
             ),
         )
-        self.scout_api: ScoutAPI = config.scout_api
+        self.scout_api_37: ScoutAPI = config.scout_api_37
+        self.scout_api_38: ScoutAPI = config.scout_api_38
         self.status_db: Store = config.status_db
         self.trailblazer_api: TrailblazerAPI = config.trailblazer_api

cg/meta/upload/scout/hk_tags.py

@@ -29,12 +29,12 @@ class CaseTags(BaseModel):
     )
     cnv_report: set[str] | None = Field(None, description="CNV visualization report for cancer")
     smn_tsv: set[str] | None = Field(None, description="SMN gene variants, only for rare disease")
-    peddy_ped: set[str] = Field(None, description="Ped info from peddy, only for rare disease")
-    peddy_sex: set[str] | None = Field(None, description="Peddy sex check, only for rare disease")
-    peddy_check: set[str] = Field(None, description="Peddy pedigree check, only for rare disease")
+    peddy_ped: set[str] = Field(None, description="Ped info from peddy")
+    peddy_sex: set[str] | None = Field(None, description="Peddy sex check")
+    peddy_check: set[str] = Field(None, description="Peddy pedigree check")
     somalier_samples: set[str] = Field(None, description="Somalier samples info")
     somalier_pairs: set[str] = Field(None, description="Somalier pairs info")
-    sex_check: set[str] = Field(None, description="Somalier sex check info, only for rare disease")
+    sex_check: set[str] = Field(None, description="Somalier sex check info")
     multiqc_report: set[str] | None = Field(None, description="MultiQC report")
     multiqc: set[str] | None = Field(None, description="MultiQC report")
     delivery_report: set[str] | None = Field(None, description="Delivery report")
@@ -74,6 +74,7 @@ class SampleTags(BaseModel):
     bam_file: set[str] | None = None
     alignment_file: set[str] | None = None
     alignment_path: set[str] | None = None
+    assembly_alignment_path: set[str] | None = None
     d4_file: set[str] | None = None
     vcf2cytosure: set[str] | None = None
     eklipse_path: set[str] | None = None
@@ -82,6 +83,8 @@ class SampleTags(BaseModel):
     chromograph_coverage: set[str] | None = None
     chromograph_regions: set[str] | None = None
     chromograph_sites: set[str] | None = None
+    hificnv_coverage: set[str] | None = None
+    minor_allele_frequency_wig: set[str] | None = None
     reviewer_alignment: set[str] | None = None
     reviewer_alignment_index: set[str] | None = None
     reviewer_catalog: set[str] | None = None

cg/meta/upload/scout/mip_config_builder.py

@@ -81,7 +81,7 @@ def build_load_config(
             load_config.samples.append(
                 self.build_config_sample(case_sample=db_sample, hk_version=hk_version)
             )
-        self.include_pedigree_picture(load_config)
+        self.include_pedigree_picture(load_config=load_config, analysis=analysis)
         return load_config
 
     def include_case_files(self, load_config: MipLoadConfig, hk_version: Version) -> None: