From 9e8f7fdd7f72e99e22e712b9da3fab7b6b8c983a Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 8 Sep 2025 06:07:50 -0700
Subject: [PATCH 01/40] Correct typo

---
 mGAP/resources/views/phenotypes.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mGAP/resources/views/phenotypes.html b/mGAP/resources/views/phenotypes.html
index e97e8e0c..eb1f0e6a 100644
--- a/mGAP/resources/views/phenotypes.html
+++ b/mGAP/resources/views/phenotypes.html
@@ -16,7 +16,7 @@
                 ['Nervous system','Vision','Coats-like retinopathy','','','Liu et al., 2015:25656754'],
                 ['Nervous system','Neurological','Batten disease','CLN7','c.769delA; p.Ile257LeufsTer36','McBride et al., 2018:30048804'],
                 ['Nervous system','Neurological','Krabbe disease','GALC','c.435_436delAC; p.Leu146fs','Luzi et al., 1997:9192853;Baskin et al., 1998:10090061', 'Hordeaux et al., 2022:35333110'],
-                ['Nervous system','Neurological','Pelizaiaeus-Merzbacher disease','PLP1','c.682 T > C; p.Cys228Arg','Sherman et al., 2021:34364975'],
+                ['Nervous system','Neurological','Pelizaeus-Merzbacher disease','PLP1','c.682 T > C; p.Cys228Arg','Sherman et al., 2021:34364975'],
                 ['Nervous system','Neurological','Epilepsy','','','Salinas et al., 2015:26290449;Akos Szabo et al., 2019:31592545'],
                 ['Nervous system','Psychiatric','Naltrexone response','OPRM1','c.77C>G; p.Pro26Arg','Vallender et al., 2010:20153935'],
                 ['Nervous system','Psychiatric','Anxiety ','5-HTT','5-HTTLPR','Spinelli  et al., 2012:22293001'],

From 37ab88963473cc32b4072c35dfad7585d3e9280f Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Tue, 9 Sep 2025 10:34:35 -0700
Subject: [PATCH 02/40] Add MCC request field for shipping

---
 .../postgresql/mcc-20.018-20.019.sql          |   1 +
 .../dbscripts/sqlserver/mcc-20.018-20.019.sql |   1 +
 mcc/resources/schemas/mcc.xml                 |   4 +
 .../client/AnimalRequest/animal-request.tsx   | 557 ++++++++++--------
 .../client/AnimalRequest/components/values.ts |   1 +
 mcc/src/client/components/RequestUtils.tsx    |   2 +
 mcc/src/org/labkey/mcc/MccModule.java         |   2 +-
 .../org/labkey/test/tests/mcc/MccTest.java    |   1 +
 8 files changed, 334 insertions(+), 235 deletions(-)
 create mode 100644 mcc/resources/schemas/dbscripts/postgresql/mcc-20.018-20.019.sql
 create mode 100644 mcc/resources/schemas/dbscripts/sqlserver/mcc-20.018-20.019.sql

diff --git a/mcc/resources/schemas/dbscripts/postgresql/mcc-20.018-20.019.sql b/mcc/resources/schemas/dbscripts/postgresql/mcc-20.018-20.019.sql
new file mode 100644
index 00000000..67506d33
--- /dev/null
+++ b/mcc/resources/schemas/dbscripts/postgresql/mcc-20.018-20.019.sql
@@ -0,0 +1 @@
+ALTER TABLE mcc.animalRequests ADD shippingAcknowledgement bool;
diff --git a/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.018-20.019.sql b/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.018-20.019.sql
new file mode 100644
index 00000000..aa155ac9
--- /dev/null
+++ b/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.018-20.019.sql
@@ -0,0 +1 @@
+ALTER TABLE mcc.animalRequests ADD shippingAcknowledgement bit;
diff --git a/mcc/resources/schemas/mcc.xml b/mcc/resources/schemas/mcc.xml
index 86970ae2..0a0403e4 100644
--- a/mcc/resources/schemas/mcc.xml
+++ b/mcc/resources/schemas/mcc.xml
@@ -373,6 +373,10 @@
                 <columnTitle>IACUC Protocol #</columnTitle>
                 <nullable>true</nullable>
             </column>
+            <column columnName="shippingAcknowledgement">
+                <columnTitle>Shipping Acknowledgement Entered?</columnTitle>
+                <nullable>true</nullable>
+            </column>
             <column columnName="comments">
                 <columnTitle>Other Comments</columnTitle>
                 <nullable>true</nullable>
diff --git a/mcc/src/client/AnimalRequest/animal-request.tsx b/mcc/src/client/AnimalRequest/animal-request.tsx
index e09acd14..fc79fd36 100644
--- a/mcc/src/client/AnimalRequest/animal-request.tsx
+++ b/mcc/src/client/AnimalRequest/animal-request.tsx
@@ -44,7 +44,8 @@ import {
     institutionTypeOptions,
     methodsProposedPlaceholder,
     signingOfficialTooltip,
-    terminalProceduresLabel
+    terminalProceduresLabel,
+    shippingAcknowledgementStatement
 } from './components/values';
 import AnimalCensus from './components/census';
 
@@ -330,6 +331,7 @@ export function AnimalRequest() {
                         "grantnumber" : data.get("funding-grant-number"),
                         "applicationduedate": data.get("funding-application-due-date"),
                         "comments": data.get("comments"),
+                        "shippingAcknowledgement": !!data.get("shippingAcknowledgement"),
                         "status": requestData.request.status,
                     }]
                 },
@@ -461,308 +463,395 @@ export function AnimalRequest() {
 
     return (
         <>
-        <form className="tw-w-full tw-max-w-4xl" onSubmit={handleSubmit} autoComplete="off" id={"animalRequestForm"} ref={formRef}>
-            <h3>Overview</h3>
+            <form className="tw-w-full tw-max-w-4xl" onSubmit={handleSubmit} autoComplete="off" id={'animalRequestForm'}
+                  ref={formRef}>
+                <h3>Overview</h3>
 
-            <Title text="1. Project Title*"/>
-            <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
-                <ErrorMessageHandler isSubmitting={isSubmitting}>
-                    <Input id="project-title" ariaLabel="Title" isSubmitting={isSubmitting} required={doEnforceRequiredFields()} placeholder="Project Title" defaultValue={requestData.request.title}/>
-                </ErrorMessageHandler>
-            </div>
-
-            <Title text="2. Project Narrative*"/>
-            <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
-                <ErrorMessageHandler isSubmitting={isSubmitting}>
-                    <TextArea id="project-narrative" ariaLabel="Project Narrative" isSubmitting={isSubmitting} placeholder="Project Narrative" required={doEnforceRequiredFields()} defaultValue={requestData.request.narrative}/>
-                </ErrorMessageHandler>
-            </div>
-
-            <Title text="3. Research/Disease Focus*"/>
-            <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
-                <ErrorMessageHandler isSubmitting={isSubmitting}>
-                    <Input id="diseasefocus" ariaLabel="Research/Disease Focus" isSubmitting={isSubmitting} required={doEnforceRequiredFields()} placeholder="What is the research area or disease focus of this project" defaultValue={requestData.request.diseasefocus}/>
-                </ErrorMessageHandler>
-            </div>
-
-            <Title text="4. How does the research relate to neuroscience?*"/>
-            <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
-                <ErrorMessageHandler isSubmitting={isSubmitting}>
-                    <TextArea id="neuroscience" ariaLabel="Connection to neuroscience" isSubmitting={isSubmitting} placeholder="How does the research relate to neuroscience" required={doEnforceRequiredFields()} defaultValue={requestData.request.neuroscience}/>
-                </ErrorMessageHandler>
-            </div>
-
-            <h3>General Information</h3>
-            <ErrorMessageHandler isSubmitting={isSubmitting}>
-            <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
-                <Title text="1. Principal Investigator*"/>
-
-                <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
-                    <Input id="investigator-last-name" ariaLabel="Last Name" isSubmitting={isSubmitting} required={doEnforceRequiredFields()} placeholder="Last Name" defaultValue={requestData.request.lastname}/>
-                </div>
-
-                <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
-                    <Input id="investigator-first-name" ariaLabel="First Name" isSubmitting={isSubmitting} required={doEnforceRequiredFields()} placeholder="First Name" defaultValue={requestData.request.firstname}/>
-                </div>
-
-                <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
-                    <Input id="investigator-middle-initial" isSubmitting={isSubmitting} required={false} placeholder="Middle Initial" defaultValue={requestData.request.middleinitial} maxLength="8"/>
-                </div>
-            </div>
-            </ErrorMessageHandler>
-
-            <ErrorMessageHandler isSubmitting={isSubmitting}>
-            <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
-                <div className="tw-relative tw-w-full tw-mb-6 md:tw-mb-0">
-                    <Title text="2. Are you an early-stage investigator?&nbsp;"/>
-                    <Tooltip id="early-stage-investigator-helper"
-                       text={earlyInvestigatorTooltip}
-                    />
-                </div>
-
-                <div className="tw-w-full tw-px-3 tw-mt-6">
-                    <YesNoRadio id="is-early-stage-investigator" ariaLabel="Early Stage Investigator" isSubmitting={isSubmitting} required={doEnforceRequiredFields()} defaultValue={requestData.request.earlystageinvestigator}/>
+                <Title text="1. Project Title*"/>
+                <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
+                    <ErrorMessageHandler isSubmitting={isSubmitting}>
+                        <Input id="project-title" ariaLabel="Title" isSubmitting={isSubmitting}
+                               required={doEnforceRequiredFields()} placeholder="Project Title"
+                               defaultValue={requestData.request.title}/>
+                    </ErrorMessageHandler>
                 </div>
-            </div>
-            </ErrorMessageHandler>
-
-            <ErrorMessageHandler isSubmitting={isSubmitting}>
-            <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
-                <Title text="3. Affiliated research institution*"/>
 
+                <Title text="2. Project Narrative*"/>
                 <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
-                    <Input id="institution-name" ariaLabel="Institution Name" isSubmitting={isSubmitting} placeholder="Name" required={doEnforceRequiredFields()} defaultValue={requestData.request.institutionname}/>
+                    <ErrorMessageHandler isSubmitting={isSubmitting}>
+                        <TextArea id="project-narrative" ariaLabel="Project Narrative" isSubmitting={isSubmitting}
+                                  placeholder="Project Narrative" required={doEnforceRequiredFields()}
+                                  defaultValue={requestData.request.narrative}/>
+                    </ErrorMessageHandler>
                 </div>
 
-                <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
-                    <Input id="institution-city" ariaLabel="Institution City" isSubmitting={isSubmitting} placeholder="City" required={doEnforceRequiredFields()} defaultValue={requestData.request.institutioncity}/>
+                <Title text="3. Research/Disease Focus*"/>
+                <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
+                    <ErrorMessageHandler isSubmitting={isSubmitting}>
+                        <Input id="diseasefocus" ariaLabel="Research/Disease Focus" isSubmitting={isSubmitting}
+                               required={doEnforceRequiredFields()}
+                               placeholder="What is the research area or disease focus of this project"
+                               defaultValue={requestData.request.diseasefocus}/>
+                    </ErrorMessageHandler>
                 </div>
 
-                <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
-                    <Input id="institution-state" ariaLabel="Institution State" isSubmitting={isSubmitting} placeholder="State" required={doEnforceRequiredFields()} defaultValue={requestData.request.institutionstate}/>
+                <Title text="4. How does the research relate to neuroscience?*"/>
+                <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
+                    <ErrorMessageHandler isSubmitting={isSubmitting}>
+                        <TextArea id="neuroscience" ariaLabel="Connection to neuroscience" isSubmitting={isSubmitting}
+                                  placeholder="How does the research relate to neuroscience"
+                                  required={doEnforceRequiredFields()} defaultValue={requestData.request.neuroscience}/>
+                    </ErrorMessageHandler>
                 </div>
 
-                <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
-                    <Input id="institution-country" ariaLabel="Institution Country" isSubmitting={isSubmitting} placeholder="Country" required={doEnforceRequiredFields()} defaultValue={requestData.request.institutioncountry}/>
-                </div>
+                <h3>General Information</h3>
+                <ErrorMessageHandler isSubmitting={isSubmitting}>
+                    <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
+                        <Title text="1. Principal Investigator*"/>
 
-                <Title text="4. Affiliated Research Institution Type*"/>
+                        <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
+                            <Input id="investigator-last-name" ariaLabel="Last Name" isSubmitting={isSubmitting}
+                                   required={doEnforceRequiredFields()} placeholder="Last Name"
+                                   defaultValue={requestData.request.lastname}/>
+                        </div>
 
-                <div className="tw-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
-                    <Select id="institution-type" ariaLabel="Institution Type" isSubmitting={isSubmitting} placeholder="Type" required={doEnforceRequiredFields()} defaultValue={requestData.request.institutiontype} options={institutionTypeOptions}/>
-                </div>
-            </div>
-            </ErrorMessageHandler>
+                        <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
+                            <Input id="investigator-first-name" ariaLabel="First Name" isSubmitting={isSubmitting}
+                                   required={doEnforceRequiredFields()} placeholder="First Name"
+                                   defaultValue={requestData.request.firstname}/>
+                        </div>
 
-            <ErrorMessageHandler isSubmitting={isSubmitting}>
-            <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
-                <div className="tw-relative tw-w-full tw-mb-6 md:tw-mb-0">
-                    <Title text="5. Institution Signing Official*&nbsp;"/>
-                    <Tooltip id="signing-official-helper"
-                         text={signingOfficialTooltip}
-                    />
-                </div>
+                        <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
+                            <Input id="investigator-middle-initial" isSubmitting={isSubmitting} required={false}
+                                   placeholder="Middle Initial" defaultValue={requestData.request.middleinitial}
+                                   maxLength="8"/>
+                        </div>
+                    </div>
+                </ErrorMessageHandler>
 
+                <ErrorMessageHandler isSubmitting={isSubmitting}>
+                    <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
+                        <div className="tw-relative tw-w-full tw-mb-6 md:tw-mb-0">
+                            <Title text="2. Are you an early-stage investigator?&nbsp;"/>
+                            <Tooltip id="early-stage-investigator-helper"
+                                     text={earlyInvestigatorTooltip}
+                            />
+                        </div>
 
-                <div className="tw-flex tw-flex-wrap tw-mt-6">
-                    <div className="tw-w-full md:tw-w-1/2 tw-px-3 tw-mb-6 md:tw-mb-0">
-                        <Input id="official-last-name" ariaLabel="Last Name" isSubmitting={isSubmitting} placeholder="Last Name" required={doEnforceRequiredFields()} defaultValue={requestData.request.officiallastname}/>
+                        <div className="tw-w-full tw-px-3 tw-mt-6">
+                            <YesNoRadio id="is-early-stage-investigator" ariaLabel="Early Stage Investigator"
+                                        isSubmitting={isSubmitting} required={doEnforceRequiredFields()}
+                                        defaultValue={requestData.request.earlystageinvestigator}/>
+                        </div>
                     </div>
+                </ErrorMessageHandler>
 
-                    <div className="tw-w-full md:tw-w-1/2 tw-px-3 tw-mb-6 md:tw-mb-0">
-                        <Input id="official-first-name" ariaLabel="First Name" isSubmitting={isSubmitting} placeholder="First Name" required={doEnforceRequiredFields()} defaultValue={requestData.request.officialfirstname}/>
-                    </div>
+                <ErrorMessageHandler isSubmitting={isSubmitting}>
+                    <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
+                        <Title text="3. Affiliated research institution*"/>
 
-                    <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
-                        <Input id="official-email" ariaLabel="Email Address" isSubmitting={isSubmitting} placeholder="Email Address" required={doEnforceRequiredFields()} defaultValue={requestData.request.officialemail}/>
-                    </div>
-                </div>
-            </div>
-            </ErrorMessageHandler>
+                        <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
+                            <Input id="institution-name" ariaLabel="Institution Name" isSubmitting={isSubmitting}
+                                   placeholder="Name" required={doEnforceRequiredFields()}
+                                   defaultValue={requestData.request.institutionname}/>
+                        </div>
 
-            <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-10">
-                <Title text="6. Co-Investigators"/>
+                        <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
+                            <Input id="institution-city" ariaLabel="Institution City" isSubmitting={isSubmitting}
+                                   placeholder="City" required={doEnforceRequiredFields()}
+                                   defaultValue={requestData.request.institutioncity}/>
+                        </div>
 
-                <CoInvestigators isSubmitting={isSubmitting} required={doEnforceRequiredFields()} coinvestigators={requestData.coinvestigators} onAddRecord={onAddInvestigator} onRemoveRecord={onRemoveCoInvestigator} />
-            </div>
+                        <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
+                            <Input id="institution-state" ariaLabel="Institution State" isSubmitting={isSubmitting}
+                                   placeholder="State" required={doEnforceRequiredFields()}
+                                   defaultValue={requestData.request.institutionstate}/>
+                        </div>
 
-            <Title text="7. Existing or proposed funding source (select all that apply)"/>
-            {/* TODO: Make into checkbox group*/}
-            <Funding id="funding" isSubmitting={isSubmitting} defaultValue={requestData.request} required={doEnforceRequiredFields()}/>
+                        <div className="tw-w-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
+                            <Input id="institution-country" ariaLabel="Institution Country" isSubmitting={isSubmitting}
+                                   placeholder="Country" required={doEnforceRequiredFields()}
+                                   defaultValue={requestData.request.institutioncountry}/>
+                        </div>
 
-            <h3>Institutional Animal Facilities and Capabilities</h3>
-            <div className="tw-w-full tw-px-3">
-                <Title text="1. Does your institution have existing NHP facilities?"/>
-                <ErrorMessageHandler isSubmitting={isSubmitting}>
-                    <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
-                        <Select id="existing-nhp-facilities" ariaLabel="Existing NHP Facilities" isSubmitting={isSubmitting} options={existingNHPFacilityOptions} defaultValue={requestData.request.existingnhpfacilities} required={doEnforceRequiredFields()}/>
+                        <Title text="4. Affiliated Research Institution Type*"/>
+
+                        <div className="tw-full md:tw-w-1/3 tw-px-3 tw-mb-6 md:tw-mb-0">
+                            <Select id="institution-type" ariaLabel="Institution Type" isSubmitting={isSubmitting}
+                                    placeholder="Type" required={doEnforceRequiredFields()}
+                                    defaultValue={requestData.request.institutiontype}
+                                    options={institutionTypeOptions}/>
+                        </div>
                     </div>
                 </ErrorMessageHandler>
 
-                <Title text="2. Does your institution have an existing marmoset colony?"/>
                 <ErrorMessageHandler isSubmitting={isSubmitting}>
                     <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
-                        <Select id="existing-marmoset-colony" ariaLabel="Existing Marmoset Colony" isSubmitting={isSubmitting} options={existingMarmosetColonyOptions} defaultValue={requestData.request.existingmarmosetcolony} required={doEnforceRequiredFields()}/>
+                        <div className="tw-relative tw-w-full tw-mb-6 md:tw-mb-0">
+                            <Title text="5. Institution Signing Official*&nbsp;"/>
+                            <Tooltip id="signing-official-helper"
+                                     text={signingOfficialTooltip}
+                            />
+                        </div>
+
+
+                        <div className="tw-flex tw-flex-wrap tw-mt-6">
+                            <div className="tw-w-full md:tw-w-1/2 tw-px-3 tw-mb-6 md:tw-mb-0">
+                                <Input id="official-last-name" ariaLabel="Last Name" isSubmitting={isSubmitting}
+                                       placeholder="Last Name" required={doEnforceRequiredFields()}
+                                       defaultValue={requestData.request.officiallastname}/>
+                            </div>
+
+                            <div className="tw-w-full md:tw-w-1/2 tw-px-3 tw-mb-6 md:tw-mb-0">
+                                <Input id="official-first-name" ariaLabel="First Name" isSubmitting={isSubmitting}
+                                       placeholder="First Name" required={doEnforceRequiredFields()}
+                                       defaultValue={requestData.request.officialfirstname}/>
+                            </div>
+
+                            <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
+                                <Input id="official-email" ariaLabel="Email Address" isSubmitting={isSubmitting}
+                                       placeholder="Email Address" required={doEnforceRequiredFields()}
+                                       defaultValue={requestData.request.officialemail}/>
+                            </div>
+                        </div>
                     </div>
                 </ErrorMessageHandler>
 
-                <Title text="3. Do you plan to breed marmosets?"/>
-                <div className="tw-w-full tw-px-3 tw-mb-4">
-                    <AnimalBreeding id="animal-breeding" isSubmitting={isSubmitting} request={requestData.request} required={doEnforceRequiredFields()}/>
+                <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-10">
+                    <Title text="6. Co-Investigators"/>
+
+                    <CoInvestigators isSubmitting={isSubmitting} required={doEnforceRequiredFields()}
+                                     coinvestigators={requestData.coinvestigators} onAddRecord={onAddInvestigator}
+                                     onRemoveRecord={onRemoveCoInvestigator}/>
                 </div>
-            </div>
 
-            <h3>Research Details</h3>
-
-            <div className="tw-flex tw-flex-wrap tw-mx-2">
-                <div className="tw-w-full tw-px-3 tw-mb-4">
-                    <Title text={"1. " + experimentalRationalePlaceholder}/>
-                    <Tooltip id="research-use-statement-helper"
-                       text={experimentalRationalePlaceholder}
-                    />
+                <Title text="7. Existing or proposed funding source (select all that apply)"/>
+                {/* TODO: Make into checkbox group*/}
+                <Funding id="funding" isSubmitting={isSubmitting} defaultValue={requestData.request}
+                         required={doEnforceRequiredFields()}/>
 
+                <h3>Institutional Animal Facilities and Capabilities</h3>
+                <div className="tw-w-full tw-px-3">
+                    <Title text="1. Does your institution have existing NHP facilities?"/>
                     <ErrorMessageHandler isSubmitting={isSubmitting}>
-                    <TextArea id="experiment-rationale" ariaLabel="Experimental rationale" isSubmitting={isSubmitting} placeholder={experimentalRationalePlaceholder} required={doEnforceRequiredFields()} defaultValue={requestData.request.experimentalrationale}/>
+                        <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
+                            <Select id="existing-nhp-facilities" ariaLabel="Existing NHP Facilities"
+                                    isSubmitting={isSubmitting} options={existingNHPFacilityOptions}
+                                    defaultValue={requestData.request.existingnhpfacilities}
+                                    required={doEnforceRequiredFields()}/>
+                        </div>
                     </ErrorMessageHandler>
-                </div>
-
-                <Title text="2. Animal Cohorts"/>
-                <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-6">
-                    <AnimalCohorts isSubmitting={isSubmitting} cohorts={requestData.cohorts} required={doEnforceRequiredFields()} onAddCohort={onAddCohort} onRemoveCohort={onRemoveCohort}/>
-                </div>
 
-                <Title text={"3. " + methodsProposedPlaceholder}/>
-                <div className="tw-w-full tw-px-3 tw-mb-6">
+                    <Title text="2. Does your institution have an existing marmoset colony?"/>
                     <ErrorMessageHandler isSubmitting={isSubmitting}>
-                    <div className="tw-w-full tw-px-3 tw-mb-6">
-                        <TextArea id="methods-proposed" ariaLabel="Methods Proposed" isSubmitting={isSubmitting} placeholder={methodsProposedPlaceholder} required={doEnforceRequiredFields()} defaultValue={requestData.request.methodsproposed}/>
-                    </div>
+                        <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
+                            <Select id="existing-marmoset-colony" ariaLabel="Existing Marmoset Colony"
+                                    isSubmitting={isSubmitting} options={existingMarmosetColonyOptions}
+                                    defaultValue={requestData.request.existingmarmosetcolony}
+                                    required={doEnforceRequiredFields()}/>
+                        </div>
                     </ErrorMessageHandler>
+
+                    <Title text="3. Do you plan to breed marmosets?"/>
+                    <div className="tw-w-full tw-px-3 tw-mb-4">
+                        <AnimalBreeding id="animal-breeding" isSubmitting={isSubmitting} request={requestData.request}
+                                        required={doEnforceRequiredFields()}/>
+                    </div>
                 </div>
 
-                <ErrorMessageHandler isSubmitting={isSubmitting}>
-                    <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
-                        <div className="tw-relative tw-w-full tw-mb-6 md:tw-mb-0">
-                            <Title text={"4. " + terminalProceduresLabel}/>
-                        </div>
+                <h3>Research Details</h3>
+
+                <div className="tw-flex tw-flex-wrap tw-mx-2">
+                    <div className="tw-w-full tw-px-3 tw-mb-4">
+                        <Title text={'1. ' + experimentalRationalePlaceholder}/>
+                        <Tooltip id="research-use-statement-helper"
+                                 text={experimentalRationalePlaceholder}
+                        />
+
+                        <ErrorMessageHandler isSubmitting={isSubmitting}>
+                            <TextArea id="experiment-rationale" ariaLabel="Experimental rationale"
+                                      isSubmitting={isSubmitting} placeholder={experimentalRationalePlaceholder}
+                                      required={doEnforceRequiredFields()}
+                                      defaultValue={requestData.request.experimentalrationale}/>
+                        </ErrorMessageHandler>
+                    </div>
 
-                        <div className="tw-w-full tw-px-3 tw-mt-6">
-                            <YesNoRadio id="is-terminalprocedures" ariaLabel="Terminal procedures" isSubmitting={isSubmitting} required={doEnforceRequiredFields()} defaultValue={requestData.request.terminalprocedures}/>
-                        </div>
+                    <Title text="2. Animal Cohorts"/>
+                    <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-6">
+                        <AnimalCohorts isSubmitting={isSubmitting} cohorts={requestData.cohorts}
+                                       required={doEnforceRequiredFields()} onAddCohort={onAddCohort}
+                                       onRemoveCohort={onRemoveCohort}/>
                     </div>
-                </ErrorMessageHandler>
 
-                <Title text={"5. " + collaborationsPlaceholder}/>
-                <div className="tw-w-full tw-px-3 tw-mb-6">
-                    <ErrorMessageHandler isSubmitting={isSubmitting}>
+                    <Title text={'3. ' + methodsProposedPlaceholder}/>
                     <div className="tw-w-full tw-px-3 tw-mb-6">
-                        <TextArea id="collaborations" ariaLabel="Collaborations" isSubmitting={isSubmitting} placeholder={collaborationsPlaceholder} required={doEnforceRequiredFields()} defaultValue={requestData.request.collaborations}/>
+                        <ErrorMessageHandler isSubmitting={isSubmitting}>
+                            <div className="tw-w-full tw-px-3 tw-mb-6">
+                                <TextArea id="methods-proposed" ariaLabel="Methods Proposed" isSubmitting={isSubmitting}
+                                          placeholder={methodsProposedPlaceholder} required={doEnforceRequiredFields()}
+                                          defaultValue={requestData.request.methodsproposed}/>
+                            </div>
+                        </ErrorMessageHandler>
                     </div>
-                    </ErrorMessageHandler>
-                </div>
 
-                <Title text={"6. " + animalWellfarePlaceholder}/>
-                <div className="tw-w-full tw-px-3 tw-mb-6">
                     <ErrorMessageHandler isSubmitting={isSubmitting}>
+                        <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
+                            <div className="tw-relative tw-w-full tw-mb-6 md:tw-mb-0">
+                                <Title text={'4. ' + terminalProceduresLabel}/>
+                            </div>
+
+                            <div className="tw-w-full tw-px-3 tw-mt-6">
+                                <YesNoRadio id="is-terminalprocedures" ariaLabel="Terminal procedures"
+                                            isSubmitting={isSubmitting} required={doEnforceRequiredFields()}
+                                            defaultValue={requestData.request.terminalprocedures}/>
+                            </div>
+                        </div>
+                    </ErrorMessageHandler>
+
+                    <Title text={'5. ' + collaborationsPlaceholder}/>
                     <div className="tw-w-full tw-px-3 tw-mb-6">
-                        <TextArea id="animal-welfare" ariaLabel="Animal Welfare" isSubmitting={isSubmitting} placeholder={animalWellfarePlaceholder} required={doEnforceRequiredFields()} defaultValue={requestData.request.animalwelfare}/>
+                        <ErrorMessageHandler isSubmitting={isSubmitting}>
+                            <div className="tw-w-full tw-px-3 tw-mb-6">
+                                <TextArea id="collaborations" ariaLabel="Collaborations" isSubmitting={isSubmitting}
+                                          placeholder={collaborationsPlaceholder} required={doEnforceRequiredFields()}
+                                          defaultValue={requestData.request.collaborations}/>
+                            </div>
+                        </ErrorMessageHandler>
                     </div>
-                    </ErrorMessageHandler>
 
-                    <ErrorMessageHandler isSubmitting={isSubmitting}>
+                    <Title text={'6. ' + animalWellfarePlaceholder}/>
                     <div className="tw-w-full tw-px-3 tw-mb-6">
-                        <input type="checkbox" name="certify" id="certify" aria-label="Certify" className={(isSubmitting ? "custom-invalid" : "")} required={doEnforceRequiredFields()} defaultChecked={requestData.request.certify}/>
-                        <label className="tw-text-gray-700 ml-1">{certificationLabel}</label>
+                        <ErrorMessageHandler isSubmitting={isSubmitting}>
+                            <div className="tw-w-full tw-px-3 tw-mb-6">
+                                <TextArea id="animal-welfare" ariaLabel="Animal Welfare" isSubmitting={isSubmitting}
+                                          placeholder={animalWellfarePlaceholder} required={doEnforceRequiredFields()}
+                                          defaultValue={requestData.request.animalwelfare}/>
+                            </div>
+                        </ErrorMessageHandler>
+
+                        <ErrorMessageHandler isSubmitting={isSubmitting}>
+                            <div className="tw-w-full tw-px-3 tw-mb-6">
+                                <input type="checkbox" name="certify" id="certify" aria-label="Certify"
+                                       className={(isSubmitting ? 'custom-invalid' : '')}
+                                       required={doEnforceRequiredFields()}
+                                       defaultChecked={requestData.request.certify}/>
+                                <label className="tw-text-gray-700 ml-1">{certificationLabel}</label>
+                            </div>
+                        </ErrorMessageHandler>
                     </div>
+
+                    <ErrorMessageHandler isSubmitting={isSubmitting}>
+                        <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
+                            <Title text="7. Attending veterinarian"/>
+
+                            <div className="tw-w-full md:tw-w-1/2 tw-px-3 tw-mb-6 md:tw-mb-0">
+                                <Input id="vet-last-name" ariaLabel="Last Name" isSubmitting={isSubmitting}
+                                       placeholder="Last Name" required={doEnforceRequiredFields()}
+                                       defaultValue={requestData.request.vetlastname}/>
+                            </div>
+
+                            <div className="tw-w-full md:tw-w-1/2 tw-px-3 tw-mb-6 md:tw-mb-0">
+                                <Input id="vet-first-name" ariaLabel="First Name" isSubmitting={isSubmitting}
+                                       placeholder="First Name" required={doEnforceRequiredFields()}
+                                       defaultValue={requestData.request.vetfirstname}/>
+                            </div>
+
+                            <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
+                                <Input id="vet-email" ariaLabel="Email" isSubmitting={isSubmitting}
+                                       placeholder="Email Address" required={doEnforceRequiredFields()}
+                                       defaultValue={requestData.request.vetemail}/>
+                            </div>
+                        </div>
                     </ErrorMessageHandler>
                 </div>
 
-                <ErrorMessageHandler isSubmitting={isSubmitting}>
-                <div className="tw-flex tw-flex-wrap tw-mx-2 tw-mb-4">
-                    <Title text="7. Attending veterinarian"/>
-
-                    <div className="tw-w-full md:tw-w-1/2 tw-px-3 tw-mb-6 md:tw-mb-0">
-                        <Input id="vet-last-name" ariaLabel="Last Name" isSubmitting={isSubmitting} placeholder="Last Name" required={doEnforceRequiredFields()} defaultValue={requestData.request.vetlastname}/>
-                    </div>
-
-                    <div className="tw-w-full md:tw-w-1/2 tw-px-3 tw-mb-6 md:tw-mb-0">
-                        <Input id="vet-first-name" ariaLabel="First Name" isSubmitting={isSubmitting} placeholder="First Name" required={doEnforceRequiredFields()} defaultValue={requestData.request.vetfirstname}/>
-                    </div>
+                <IACUCProtocol id="iacuc" isSubmitting={isSubmitting} required={doEnforceRequiredFields()}
+                               request={requestData.request}/>
 
-                    <div className="tw-w-full tw-px-3 tw-mb-6 md:tw-mb-0">
-                        <Input id="vet-email" ariaLabel="Email" isSubmitting={isSubmitting} placeholder="Email Address" required={doEnforceRequiredFields()} defaultValue={requestData.request.vetemail}/>
-                    </div>
+                <div className="tw-relative tw-w-full tw-mb-6 md:tw-mb-0">
+                    <Title text="9. Will participate in the MCC Census?&nbsp;"/>
+                    <Tooltip id="census-helper" text={censusToolTip}/>
+                </div>
+                <div className="tw-w-full tw-px-3 tw-mt-3">
+                    <AnimalCensus id="census" isSubmitting={isSubmitting} required={doEnforceRequiredFields()}
+                                  request={requestData.request}/>
                 </div>
-                </ErrorMessageHandler>
-            </div>
-
-            <IACUCProtocol id="iacuc" isSubmitting={isSubmitting} required={doEnforceRequiredFields()} request={requestData.request}/>
 
-            <div className="tw-relative tw-w-full tw-mb-6 md:tw-mb-0">
-                <Title text="9. Will participate in the MCC Census?&nbsp;"/>
-                <Tooltip id="census-helper" text={censusToolTip}/>
-            </div>
-            <div className="tw-w-full tw-px-3 tw-mt-3">
-                <AnimalCensus id="census" isSubmitting={isSubmitting} required={doEnforceRequiredFields()} request={requestData.request}/>
-            </div>
+                <div className="tw-relative tw-w-full tw-mb-6 md:tw-mb-4">
+                    <Title text="10. Shipment Acknowledgement&nbsp;"/>
+                </div>
 
-            <Title text={"10. " + commentsPlaceholder}/>
-            <div className="tw-w-full tw-px-3 tw-mb-6">
                 <ErrorMessageHandler isSubmitting={isSubmitting}>
-                    <div className="tw-w-full tw-px-3 tw-mb-6">
-                        <TextArea id="comments" ariaLabel="Comments" isSubmitting={isSubmitting} placeholder={commentsPlaceholder} required={false} defaultValue={requestData.request.comments}/>
+                    <div className="tw-w-full tw-px-6 tw-mb-6">
+                        {shippingAcknowledgementStatement}
+                        <p />
+                        <input type="checkbox" name="shippingAcknowledgement" id="shippingAcknowledgement" aria-label="Shipping Acknowledgement"
+                               className={(isSubmitting ? 'custom-invalid' : '')}
+                               required={doEnforceRequiredFields()}
+                               defaultChecked={requestData.request.shippingAcknowledgement}/>
+                        <label className="tw-text-gray-700 ml-1">I acknowledge this statement</label>
                     </div>
                 </ErrorMessageHandler>
-            </div>
 
-            <div className="tw-flex tw-flex-wrap tw-mx-2">
-                <Title text="Request Status: "/>{requestData.request.status}
-            </div>
+                <Title text={'11. ' + commentsPlaceholder}/>
+                <div className="tw-w-full tw-px-3 tw-mb-6">
+                    <ErrorMessageHandler isSubmitting={isSubmitting}>
+                        <div className="tw-w-full tw-px-3 tw-mb-6">
+                            <TextArea id="comments" ariaLabel="Comments" isSubmitting={isSubmitting}
+                                      placeholder={commentsPlaceholder} required={false}
+                                      defaultValue={requestData.request.comments}/>
+                        </div>
+                    </ErrorMessageHandler>
+                </div>
 
-            <div className="tw-flex tw-flex-wrap tw-mx-2">
-                <Button baseColor="red" marginLeft="auto" text="Cancel" onClick={(e) => {
-                    e.preventDefault()
+                <div className="tw-flex tw-flex-wrap tw-mx-2">
+                    <Title text="Request Status: "/>{requestData.request.status}
+                </div>
 
-                    if (confirm("You are about to leave this page.")) {
-                        window.location.href = ActionURL.buildURL('mcc', 'mccRequests.view')
-                    }
-                }} />
+                <div className="tw-flex tw-flex-wrap tw-mx-2">
+                    <Button baseColor="red" marginLeft="auto" text="Cancel" onClick={(e) => {
+                        e.preventDefault();
 
-                <Button onClick={(e) => {
-                    handleSubmitButton(e, false);
-                 }} text={getSaveButtonText()} display={hasEditPermission()}/>
+                        if (confirm('You are about to leave this page.')) {
+                            window.location.href = ActionURL.buildURL('mcc', 'mccRequests.view');
+                        }
+                    }}/>
 
-                <Button onClick={(e) => {
-                    handleSubmitButton(e, true);
-                 }} text={getSubmitButtonText()} display={hasEditPermission()}/>
+                    <Button onClick={(e) => {
+                        handleSubmitButton(e, false);
+                    }} text={getSaveButtonText()} display={hasEditPermission()}/>
 
-                <Button onClick={(e) => {
-                    e.preventDefault()
-                    setShowWithdrawDialog(true)
-                }} text={"Withdraw"} display={shouldShowWithdraw()}/>
-            </div>
-        </form>
-
-        <SavingOverlay display={displayOverlay} />
-
-        <Dialog open={showWithdrawDialog}>
-            <DialogTitle>Withdraw Request</DialogTitle>
-            <DialogContent>
-                <DialogContentText>Please enter a reason for withdrawing this request</DialogContentText>
-                <TextareaAutosize
-                    minRows={4}
-                    id="withdrawReason"
-                    required={true}
-                    autoFocus={true}
-                    defaultValue={withdrawReasonText}
-                    form={"animalRequestForm"}
-                    onChange={(e) => setWithdrawReasonText(e.target.value)}
-                />
-            </DialogContent>
-            <DialogActions>
-                <Box mr="5px">
                     <Button onClick={(e) => {
-                        if (!withdrawReasonText) {
+                        handleSubmitButton(e, true);
+                    }} text={getSubmitButtonText()} display={hasEditPermission()}/>
+
+                    <Button onClick={(e) => {
+                        e.preventDefault();
+                        setShowWithdrawDialog(true);
+                    }} text={'Withdraw'} display={shouldShowWithdraw()}/>
+                </div>
+            </form>
+
+            <SavingOverlay display={displayOverlay}/>
+
+            <Dialog open={showWithdrawDialog}>
+                <DialogTitle>Withdraw Request</DialogTitle>
+                <DialogContent>
+                    <DialogContentText>Please enter a reason for withdrawing this request</DialogContentText>
+                    <TextareaAutosize
+                        minRows={4}
+                        id="withdrawReason"
+                        required={true}
+                        autoFocus={true}
+                        defaultValue={withdrawReasonText}
+                        form={'animalRequestForm'}
+                        onChange={(e) => setWithdrawReasonText(e.target.value)}
+                    />
+                </DialogContent>
+                <DialogActions>
+                    <Box mr="5px">
+                        <Button onClick={(e) => {
+                            if (!withdrawReasonText) {
                             alert("Must enter the reason")
                             return
                         }
diff --git a/mcc/src/client/AnimalRequest/components/values.ts b/mcc/src/client/AnimalRequest/components/values.ts
index 055ea750..afb77137 100644
--- a/mcc/src/client/AnimalRequest/components/values.ts
+++ b/mcc/src/client/AnimalRequest/components/values.ts
@@ -14,6 +14,7 @@ export const animalWellfarePlaceholder = "Animal welfare (proposed care and use)
 export const censusReasonPlaceholder = "Reason for not participating"
 
 export const certificationLabel = "I certify and I have obtained approval for this study from my institution."
+export const shippingAcknowledgementStatement = "I will be ready to receive animals within 60 days of approval, provided that they are available from a breeding center.  I understand that failure to do so will result in per diem charges billed to me."
 export const terminalProceduresLabel = "Includes terminal procedures?"
 
 export const fundingSourceOptions = [
diff --git a/mcc/src/client/components/RequestUtils.tsx b/mcc/src/client/components/RequestUtils.tsx
index b25c4a8e..4bca1478 100644
--- a/mcc/src/client/components/RequestUtils.tsx
+++ b/mcc/src/client/components/RequestUtils.tsx
@@ -48,6 +48,7 @@ export class AnimalRequestProps {
     vetlastname: string;
     vetemail: string;
     vetfirstname: string;
+    shippingAcknowledgement: boolean;
     objectid: string;
     comments: string;
 }
@@ -145,6 +146,7 @@ export async function queryRequestInformation(requestId, handleFailure) {
                 "iacucprotocol",
                 "grantnumber",
                 "applicationduedate",
+                "shippingAcknowledgement",
                 "comments",
                 "status"
             ],
diff --git a/mcc/src/org/labkey/mcc/MccModule.java b/mcc/src/org/labkey/mcc/MccModule.java
index c5217661..11292760 100644
--- a/mcc/src/org/labkey/mcc/MccModule.java
+++ b/mcc/src/org/labkey/mcc/MccModule.java
@@ -77,7 +77,7 @@ public String getName()
     @Override
     public @Nullable Double getSchemaVersion()
     {
-        return 20.018;
+        return 20.019;
     }
 
     @Override
diff --git a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java
index 300e7823..9c5caf4f 100644
--- a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java
+++ b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java
@@ -463,6 +463,7 @@ else if ("radio".equals(inputType))
             new FormElement("existing-nhp-facilities", "existingnhpfacilities", "Existing NHP facilities").select("existing"),
             new FormElement("animal-welfare", "animalwelfare", "welfare").inputType("textarea"),
             new FormElement("certify", "certify", true).checkBox(),
+            new FormElement("shippingAcknowledgement", "shippingAcknowledgement", true).checkBox(),
             new FormElement("vet-last-name", "vetlastname", "vet last name"),
             new FormElement("vet-first-name", "vetfirstname", "vet first name"),
             new FormElement("vet-email", "vetemail", "vet@email.com"),

From 24fe62d339c1cdd45f3e79e9b2994dd1229ea29f Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 12 Sep 2025 10:07:35 -0700
Subject: [PATCH 03/40] Minor code cleanup

---
 SivStudies/resources/queries/study/samples/.qview.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/SivStudies/resources/queries/study/samples/.qview.xml b/SivStudies/resources/queries/study/samples/.qview.xml
index 830bf244..8581fec3 100644
--- a/SivStudies/resources/queries/study/samples/.qview.xml
+++ b/SivStudies/resources/queries/study/samples/.qview.xml
@@ -2,6 +2,7 @@
     <columns>
         <column name="Id"/>
         <column name="date"/>
+        <column name="timePostSivChallenge/daysPostInfection"/>
         <column name="sampleType"/>
         <column name="quantity"/>
         <column name="quantity_units"/>

From 01a05d60890ad3a581cb985ce0228f2f7106a574 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 15 Sep 2025 15:01:31 -0700
Subject: [PATCH 04/40] Improve sample query

---
 IDR/resources/queries/bimber_data/idrSampleSource.sql | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/IDR/resources/queries/bimber_data/idrSampleSource.sql b/IDR/resources/queries/bimber_data/idrSampleSource.sql
index 0a38d53a..e9a7baa6 100644
--- a/IDR/resources/queries/bimber_data/idrSampleSource.sql
+++ b/IDR/resources/queries/bimber_data/idrSampleSource.sql
@@ -18,7 +18,12 @@ SELECT
 Rh as Id,
 ID as sampleid,
 SampleDate as date,
-Tissue as sampleType,
+CASE
+    WHEN (Tissue IS NOT NULL AND Tissue != '') AND (SampleType IS NOT NULL AND SampleType != '') THEN (SampleType || ' / ' || Tissue)
+    WHEN (Tissue IS NOT NULL AND Tissue = '') THEN Tissue
+    WHEN (SampleType IS NOT NULL AND Tissue = '') THEN SampleType
+    ELSE NULL
+END AS sampleType,
 null as quantity,
 
 'Hansen/IDR' as dataSource

From 9edb28bdaf08b4c1136404f7c2e9188d9cd5a6e3 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Tue, 16 Sep 2025 06:48:57 -0700
Subject: [PATCH 05/40] Add CSP exception

---
 mGAP/src/org/labkey/mgap/mGAPModule.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mGAP/src/org/labkey/mgap/mGAPModule.java b/mGAP/src/org/labkey/mgap/mGAPModule.java
index 9375e0a7..3799f84c 100644
--- a/mGAP/src/org/labkey/mgap/mGAPModule.java
+++ b/mGAP/src/org/labkey/mgap/mGAPModule.java
@@ -111,6 +111,7 @@ public void doStartupAfterSpringConfig(ModuleContext moduleContext)
         ContentSecurityPolicyFilter.registerAllowedSources(this.getClass().getName(), Directive.Connection, "https://code.jquery.com", "https://*.fontawesome.com");
         ContentSecurityPolicyFilter.registerAllowedSources(this.getClass().getName(), Directive.Style, "https://code.jquery.com", "https://www.gstatic.com");
         ContentSecurityPolicyFilter.registerAllowedSources(this.getClass().getName(), Directive.Font, "https://*.fontawesome.com");
+        ContentSecurityPolicyFilter.registerAllowedSources(this.getClass().getName(), Directive.Connection, "https://oss.maxcdn.com");
 
         new PipelineStartup();
     }

From 583865c59fd2c8f6fd740a8f23b488046a1a4106 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Wed, 17 Sep 2025 11:42:05 -0700
Subject: [PATCH 06/40] Update outcome SQL

---
 IDR/resources/queries/bimber_data/idrOutcomeSource.sql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/IDR/resources/queries/bimber_data/idrOutcomeSource.sql b/IDR/resources/queries/bimber_data/idrOutcomeSource.sql
index 7621857c..d254000c 100644
--- a/IDR/resources/queries/bimber_data/idrOutcomeSource.sql
+++ b/IDR/resources/queries/bimber_data/idrOutcomeSource.sql
@@ -6,6 +6,7 @@ cohortStart as date,
 CASE
     WHEN contprog = 'C' THEN 'Controller'
     WHEN contprog = 'P' THEN 'Progressor'
+    ELSE contprog
 END as outcome,
 
 'Hansen/IDR' as dataSource

From c2a6067b1efa410b73bd348327a864909d3f879b Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 18 Sep 2025 09:41:14 -0700
Subject: [PATCH 07/40] Add comments columns

---
 .../study/additionalDatatypes.query.xml       |  3 +++
 .../study/additionalDatatypes/.qview.xml      |  1 +
 .../resources/queries/study/flow.query.xml    |  2 +-
 .../resources/queries/study/flow/.qview.xml   |  1 +
 .../queries/study/genetics.query.xml          |  4 +++
 .../queries/study/genetics/.qview.xml         |  1 +
 .../queries/study/immunizations.query.xml     |  4 +++
 .../queries/study/immunizations/.qview.xml    |  1 +
 .../resources/queries/study/labwork.query.xml |  4 +++
 .../queries/study/labwork/.qview.xml          |  1 +
 .../queries/study/procedures.query.xml        |  4 +++
 .../queries/study/procedures/.qview.xml       |  1 +
 .../resources/queries/study/samples.query.xml |  4 +++
 .../queries/study/samples/.qview.xml          |  1 +
 .../queries/study/treatments.query.xml        |  4 +++
 .../queries/study/treatments/.qview.xml       |  1 +
 .../queries/study/viralLoads/.qview.xml       |  1 +
 .../queries/study/viralloads.query.xml        |  4 +++
 .../study/datasets/datasets_metadata.xml      | 26 ++++++++++++++++++-
 19 files changed, 66 insertions(+), 2 deletions(-)

diff --git a/SivStudies/resources/queries/study/additionalDatatypes.query.xml b/SivStudies/resources/queries/study/additionalDatatypes.query.xml
index 1b446dfd..45261263 100644
--- a/SivStudies/resources/queries/study/additionalDatatypes.query.xml
+++ b/SivStudies/resources/queries/study/additionalDatatypes.query.xml
@@ -18,6 +18,9 @@
                     <column columnName="description">
                         <columnTitle>Description</columnTitle>
                     </column>
+                    <column columnName="comments">
+                        <columnTitle>Comments</columnTitle>
+                    </column>
                 </columns>
             </table>
         </tables>
diff --git a/SivStudies/resources/queries/study/additionalDatatypes/.qview.xml b/SivStudies/resources/queries/study/additionalDatatypes/.qview.xml
index 612c76c8..46110c9f 100644
--- a/SivStudies/resources/queries/study/additionalDatatypes/.qview.xml
+++ b/SivStudies/resources/queries/study/additionalDatatypes/.qview.xml
@@ -5,6 +5,7 @@
         <column name="timePostSivChallenge/daysPostInfection"/>
         <column name="category"/>
         <column name="description"/>
+        <column name="comments"/>
         <column name="dataSource"/>
     </columns>
     <sorts>
diff --git a/SivStudies/resources/queries/study/flow.query.xml b/SivStudies/resources/queries/study/flow.query.xml
index 63f49ed2..63a7946a 100644
--- a/SivStudies/resources/queries/study/flow.query.xml
+++ b/SivStudies/resources/queries/study/flow.query.xml
@@ -24,7 +24,7 @@
                     <column columnName="units">
                         <columnTitle>Units</columnTitle>
                     </column>
-                    <column columnName="comment">
+                    <column columnName="comments">
                         <columnTitle>Comments</columnTitle>
                     </column>
                     <column columnName="dataSource">
diff --git a/SivStudies/resources/queries/study/flow/.qview.xml b/SivStudies/resources/queries/study/flow/.qview.xml
index c00809cd..7c9abe0b 100644
--- a/SivStudies/resources/queries/study/flow/.qview.xml
+++ b/SivStudies/resources/queries/study/flow/.qview.xml
@@ -7,6 +7,7 @@
         <column name="population"/>
         <column name="result"/>
         <column name="units"/>
+        <column name="comments"/>
         <column name="dataSource"/>
         <column name="viralLoad"/>
     </columns>
diff --git a/SivStudies/resources/queries/study/genetics.query.xml b/SivStudies/resources/queries/study/genetics.query.xml
index 93794571..334a603e 100644
--- a/SivStudies/resources/queries/study/genetics.query.xml
+++ b/SivStudies/resources/queries/study/genetics.query.xml
@@ -23,6 +23,10 @@
                     <column columnName="score">
                         <columnTitle>Score</columnTitle>
                     </column>
+                    <column columnName="comments">
+                        <columnTitle>Comments</columnTitle>
+                        <inputType>textarea</inputType>
+                    </column>
                     <column columnName="dataSource">
                         <columnTitle>Data Source</columnTitle>
                     </column>
diff --git a/SivStudies/resources/queries/study/genetics/.qview.xml b/SivStudies/resources/queries/study/genetics/.qview.xml
index bb737b81..0a932149 100644
--- a/SivStudies/resources/queries/study/genetics/.qview.xml
+++ b/SivStudies/resources/queries/study/genetics/.qview.xml
@@ -7,6 +7,7 @@
         <column name="marker"/>
         <column name="result"/>
         <column name="score"/>
+        <column name="comments"/>
         <column name="dataSource"/>
     </columns>
     <sorts>
diff --git a/SivStudies/resources/queries/study/immunizations.query.xml b/SivStudies/resources/queries/study/immunizations.query.xml
index 8e2c1528..7e1a7a60 100644
--- a/SivStudies/resources/queries/study/immunizations.query.xml
+++ b/SivStudies/resources/queries/study/immunizations.query.xml
@@ -27,6 +27,10 @@
                     <column columnName="reason">
                         <columnTitle>Reason</columnTitle>
                     </column>
+                    <column columnName="comments">
+                        <columnTitle>Comments</columnTitle>
+                        <inputType>textarea</inputType>
+                    </column>
                     <column columnName="dataSource">
                         <columnTitle>Data Source</columnTitle>
                     </column>
diff --git a/SivStudies/resources/queries/study/immunizations/.qview.xml b/SivStudies/resources/queries/study/immunizations/.qview.xml
index 8264aae3..575150b0 100644
--- a/SivStudies/resources/queries/study/immunizations/.qview.xml
+++ b/SivStudies/resources/queries/study/immunizations/.qview.xml
@@ -7,6 +7,7 @@
         <column name="route"/>
         <column name="quantity"/>
         <column name="quantity_units"/>
+        <column name="comments"/>
         <column name="dataSource"/>
     </columns>
     <sorts>
diff --git a/SivStudies/resources/queries/study/labwork.query.xml b/SivStudies/resources/queries/study/labwork.query.xml
index c0808ad1..1cb7b402 100644
--- a/SivStudies/resources/queries/study/labwork.query.xml
+++ b/SivStudies/resources/queries/study/labwork.query.xml
@@ -34,6 +34,10 @@
                     <column columnName="method">
                         <columnTitle>Method</columnTitle>
                     </column>
+                    <column columnName="comments">
+                        <columnTitle>Comments</columnTitle>
+                        <inputType>textarea</inputType>
+                    </column>
                     <column columnName="dataSource">
                         <columnTitle>Data Source</columnTitle>
                     </column>
diff --git a/SivStudies/resources/queries/study/labwork/.qview.xml b/SivStudies/resources/queries/study/labwork/.qview.xml
index 55367247..80001295 100644
--- a/SivStudies/resources/queries/study/labwork/.qview.xml
+++ b/SivStudies/resources/queries/study/labwork/.qview.xml
@@ -7,6 +7,7 @@
         <column name="result"/>
         <column name="units"/>
         <column name="method"/>
+        <column name="comments"/>
         <column name="dataSource"/>
         <column name="viralLoad"/>
         <column name="ageAtTime/AgeAtTime"/>
diff --git a/SivStudies/resources/queries/study/procedures.query.xml b/SivStudies/resources/queries/study/procedures.query.xml
index eaa6a960..1f5446fe 100644
--- a/SivStudies/resources/queries/study/procedures.query.xml
+++ b/SivStudies/resources/queries/study/procedures.query.xml
@@ -15,6 +15,10 @@
                     <column columnName="procedure">
                         <columnTitle>Procedure</columnTitle>
                     </column>
+                    <column columnName="comments">
+                        <columnTitle>Comments</columnTitle>
+                        <inputType>textarea</inputType>
+                    </column>
                     <column columnName="dataSource">
                         <columnTitle>Data Source</columnTitle>
                     </column>
diff --git a/SivStudies/resources/queries/study/procedures/.qview.xml b/SivStudies/resources/queries/study/procedures/.qview.xml
index 01c43bfd..a7fb29c4 100644
--- a/SivStudies/resources/queries/study/procedures/.qview.xml
+++ b/SivStudies/resources/queries/study/procedures/.qview.xml
@@ -4,6 +4,7 @@
         <column name="date"/>
         <column name="category"/>
         <column name="procedure"/>
+        <column name="comments"/>
         <column name="dataSource"/>
         <column name="viralLoad"/>
     </columns>
diff --git a/SivStudies/resources/queries/study/samples.query.xml b/SivStudies/resources/queries/study/samples.query.xml
index a65518ef..3aaacd32 100644
--- a/SivStudies/resources/queries/study/samples.query.xml
+++ b/SivStudies/resources/queries/study/samples.query.xml
@@ -23,6 +23,10 @@
                     <column columnName="quantity_units">
                         <columnTitle>Quantity Units</columnTitle>
                     </column>
+                    <column columnName="comments">
+                        <columnTitle>Comments</columnTitle>
+                        <inputType>textarea</inputType>
+                    </column>
                     <column columnName="dataSource">
                         <columnTitle>Data Source</columnTitle>
                     </column>
diff --git a/SivStudies/resources/queries/study/samples/.qview.xml b/SivStudies/resources/queries/study/samples/.qview.xml
index 8581fec3..0e70ce34 100644
--- a/SivStudies/resources/queries/study/samples/.qview.xml
+++ b/SivStudies/resources/queries/study/samples/.qview.xml
@@ -7,6 +7,7 @@
         <column name="quantity"/>
         <column name="quantity_units"/>
         <column name="sampleId"/>
+        <column name="comments"/>
         <column name="dataSource"/>
         <column name="viralLoad"/>
         <column name="ageAtTime/AgeAtTime"/>
diff --git a/SivStudies/resources/queries/study/treatments.query.xml b/SivStudies/resources/queries/study/treatments.query.xml
index 52a3ba76..b053767c 100644
--- a/SivStudies/resources/queries/study/treatments.query.xml
+++ b/SivStudies/resources/queries/study/treatments.query.xml
@@ -65,6 +65,10 @@
                         <isHidden>true</isHidden>
                         <shownInDetailsView>false</shownInDetailsView>
                     </column>
+                    <column columnName="comments">
+                        <columnTitle>Comments</columnTitle>
+                        <inputType>textarea</inputType>
+                    </column>
                     <column columnName="dataSource">
                         <columnTitle>Data Source</columnTitle>
                     </column>
diff --git a/SivStudies/resources/queries/study/treatments/.qview.xml b/SivStudies/resources/queries/study/treatments/.qview.xml
index d9b12386..070ec43c 100644
--- a/SivStudies/resources/queries/study/treatments/.qview.xml
+++ b/SivStudies/resources/queries/study/treatments/.qview.xml
@@ -8,6 +8,7 @@
         <column name="route"/>
         <column name="amount"/>
         <column name="amount_units"/>
+        <column name="comments"/>
         <column name="dataSource"/>
         <column name="viralLoad"/>
     </columns>
diff --git a/SivStudies/resources/queries/study/viralLoads/.qview.xml b/SivStudies/resources/queries/study/viralLoads/.qview.xml
index ef089d8f..3be12098 100644
--- a/SivStudies/resources/queries/study/viralLoads/.qview.xml
+++ b/SivStudies/resources/queries/study/viralLoads/.qview.xml
@@ -9,6 +9,7 @@
         <column name="result"/>
         <column name="units"/>
         <column name="lod"/>
+        <column name="comments"/>
         <column name="dataSource"/>
         <column name="artInformation/daysPostArtInitiation"/>
         <column name="artInformation/daysPostArtRelease"/>
diff --git a/SivStudies/resources/queries/study/viralloads.query.xml b/SivStudies/resources/queries/study/viralloads.query.xml
index 4e9e9af4..a53069a2 100644
--- a/SivStudies/resources/queries/study/viralloads.query.xml
+++ b/SivStudies/resources/queries/study/viralloads.query.xml
@@ -39,6 +39,10 @@
                         <isHidden>true</isHidden>
                         <shownInDetailsView>false</shownInDetailsView>
                     </column>
+                    <column columnName="comments">
+                        <columnTitle>Comments</columnTitle>
+                        <inputType>textarea</inputType>
+                    </column>
                     <column columnName="dataSource">
                         <columnTitle>Data Source</columnTitle>
                     </column>
diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
index 5eb90a75..18199462 100644
--- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
+++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
@@ -87,6 +87,9 @@
             <column columnName="reason">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="comments">
+                <datatype>varchar</datatype>
+            </column>
         </columns>
         <tableTitle>Medications/Treatments</tableTitle>
     </table>
@@ -126,6 +129,9 @@
             <column columnName="reason">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="comments">
+                <datatype>varchar</datatype>
+            </column>
         </columns>
         <tableTitle>Immunizations</tableTitle>
     </table>
@@ -199,6 +205,9 @@
             <column columnName="qualresult">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="comments">
+                <datatype>varchar</datatype>
+            </column>
         </columns>
         <tableTitle>Viral Loads</tableTitle>
     </table>
@@ -240,6 +249,9 @@
             <column columnName="method">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="comments">
+                <datatype>varchar</datatype>
+            </column>
         </columns>
         <tableTitle>Lab Results</tableTitle>
     </table>
@@ -360,6 +372,9 @@
             <column columnName="quantity_units">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="comments">
+                <datatype>varchar</datatype>
+            </column>
         </columns>
         <tableTitle>Samples</tableTitle>
     </table>
@@ -395,6 +410,9 @@
             <column columnName="score">
                 <datatype>double</datatype>
             </column>
+            <column columnName="comments">
+                <datatype>varchar</datatype>
+            </column>
         </columns>
         <tableTitle>Genetic Data</tableTitle>
     </table>
@@ -421,6 +439,9 @@
             <column columnName="procedure">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="comments">
+                <datatype>varchar</datatype>
+            </column>
         </columns>
         <tableTitle>Procedures</tableTitle>
     </table>
@@ -447,6 +468,9 @@
             <column columnName="description">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="comments">
+                <datatype>varchar</datatype>
+            </column>
         </columns>
         <tableTitle>Additional Datatypes</tableTitle>
     </table>
@@ -482,7 +506,7 @@
             <column columnName="units">
                 <datatype>varchar</datatype>
             </column>
-            <column columnName="comment">
+            <column columnName="comments">
                 <datatype>varchar</datatype>
             </column>
         </columns>

From ff1baa5412e12f52fa70645a3abeeeea85ad5a8c Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 18 Sep 2025 10:37:37 -0700
Subject: [PATCH 08/40] Add ETL to automatically create subjects for SIV
 studies

---
 SivStudies/resources/etls/idr-subjects.xml    | 15 +++
 .../sivstudies/etl/AddMissingIdrSubjects.java | 98 +++++++++++++++++++
 2 files changed, 113 insertions(+)
 create mode 100644 SivStudies/resources/etls/idr-subjects.xml
 create mode 100644 SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java

diff --git a/SivStudies/resources/etls/idr-subjects.xml b/SivStudies/resources/etls/idr-subjects.xml
new file mode 100644
index 00000000..fd084bea
--- /dev/null
+++ b/SivStudies/resources/etls/idr-subjects.xml
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<etl xmlns="http://labkey.org/etl/xml">
+    <name>Hansen/IDR Subjects</name>
+    <description>Hansen/IDR Subjects</description>
+    <transforms>
+        <transform id="subjects" type="TaskRefTransformStep">
+            <taskref ref="org.labkey.sivstudies.etl.AddMissingIdrSubjects">
+
+            </taskref>
+        </transform>
+    </transforms>
+    <schedule>
+        <cron expression="0 15 20 * * ?"/>
+    </schedule>
+</etl>
diff --git a/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java b/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java
new file mode 100644
index 00000000..c0873e31
--- /dev/null
+++ b/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java
@@ -0,0 +1,98 @@
+package org.labkey.sivstudies.etl;
+
+import org.apache.xmlbeans.XmlException;
+import org.jetbrains.annotations.NotNull;
+import org.labkey.api.data.Container;
+import org.labkey.api.data.ContainerManager;
+import org.labkey.api.data.TableSelector;
+import org.labkey.api.di.TaskRefTask;
+import org.labkey.api.pipeline.PipelineJob;
+import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.pipeline.RecordedActionSet;
+import org.labkey.api.query.BatchValidationException;
+import org.labkey.api.query.DuplicateKeyException;
+import org.labkey.api.query.QueryService;
+import org.labkey.api.query.QueryUpdateServiceException;
+import org.labkey.api.query.UserSchema;
+import org.labkey.api.util.PageFlowUtil;
+import org.labkey.api.writer.ContainerUser;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class AddMissingIdrSubjects implements TaskRefTask
+{
+    protected ContainerUser _containerUser;
+
+    @Override
+    public RecordedActionSet run(@NotNull PipelineJob pipelineJob) throws PipelineJobException
+    {
+        // Find existing IDs:
+        UserSchema us = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), "study");
+        if (us == null)
+        {
+            throw new PipelineJobException("Missing study schema");
+        }
+
+        List<String> existingIds = new ArrayList<>(new TableSelector(us.getTable("demographics"), PageFlowUtil.set("Id"), null, null).getArrayList(String.class));
+
+        // Source IDs:
+        Container sourceContainer = ContainerManager.getForPath("Labs/Bimber");
+        UserSchema us2 = QueryService.get().getUserSchema(_containerUser.getUser(), sourceContainer, "bimber_data");
+        if (us2 == null)
+        {
+            throw new PipelineJobException("Missing bimber_data schema");
+        }
+
+        List<String> allIds = new ArrayList<>(new TableSelector(us2.getTable("subjects"), PageFlowUtil.set("Rh"), null, null).getArrayList(String.class));
+
+        allIds.removeAll(existingIds);
+
+        if (allIds.isEmpty())
+        {
+            return null;
+        }
+
+        pipelineJob.getLogger().info("Creating {} subjects", allIds.size());
+        List<Map<String, Object>> toInsert = new ArrayList<>();
+        allIds.forEach(id -> {
+            toInsert.add(Map.of("Id", id));
+        });
+
+        try
+        {
+            BatchValidationException bve = new BatchValidationException();
+            us.getTable("demographics").getUpdateService().insertRows(_containerUser.getUser(), _containerUser.getContainer(), toInsert, bve, null, null);
+            if (bve.hasErrors())
+            {
+                throw bve;
+            }
+        }
+        catch (BatchValidationException | SQLException | DuplicateKeyException | QueryUpdateServiceException e)
+        {
+            throw new PipelineJobException(e);
+        }
+
+        return null;
+    }
+
+    @Override
+    public List<String> getRequiredSettings()
+    {
+        return List.of();
+    }
+
+    @Override
+    public void setSettings(Map<String, String> map) throws XmlException
+    {
+
+    }
+
+    @Override
+    public void setContainerUser(ContainerUser containerUser)
+    {
+        _containerUser = containerUser;
+    }
+}

From 7889a368fb9e62abedae8d16755054287a99fbd5 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 18 Sep 2025 12:45:11 -0700
Subject: [PATCH 09/40] Bugfix to sample SQL and augment ETL

---
 .../queries/bimber_data/idrSampleSource.sql        | 14 ++++++++++----
 SivStudies/resources/etls/idr-data.xml             |  2 +-
 .../resources/queries/study/samples.query.xml      | 12 ++++++++++++
 .../resources/queries/study/samples/.qview.xml     |  4 ++++
 .../study/datasets/datasets_metadata.xml           | 12 ++++++++++++
 5 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/IDR/resources/queries/bimber_data/idrSampleSource.sql b/IDR/resources/queries/bimber_data/idrSampleSource.sql
index e9a7baa6..bed1a30a 100644
--- a/IDR/resources/queries/bimber_data/idrSampleSource.sql
+++ b/IDR/resources/queries/bimber_data/idrSampleSource.sql
@@ -5,7 +5,10 @@ ID as sampleid,
 SampleDate as date,
 Tissue as sampleType,
 CellCnt as quantity,
-
+freezer,
+rack,
+box,
+"position",
 'Hansen/IDR' as dataSource
 
 FROM bimber_data.ln_loc
@@ -20,12 +23,15 @@ ID as sampleid,
 SampleDate as date,
 CASE
     WHEN (Tissue IS NOT NULL AND Tissue != '') AND (SampleType IS NOT NULL AND SampleType != '') THEN (SampleType || ' / ' || Tissue)
-    WHEN (Tissue IS NOT NULL AND Tissue = '') THEN Tissue
-    WHEN (SampleType IS NOT NULL AND Tissue = '') THEN SampleType
+    WHEN (Tissue IS NOT NULL AND Tissue != '') THEN Tissue
+    WHEN (SampleType IS NOT NULL AND SampleType != '') THEN SampleType
     ELSE NULL
 END AS sampleType,
 null as quantity,
-
+freezer,
+rack,
+box,
+"position",
 'Hansen/IDR' as dataSource
 
 FROM bimber_data.ult_loc
diff --git a/SivStudies/resources/etls/idr-data.xml b/SivStudies/resources/etls/idr-data.xml
index aa9aaf08..066afee5 100644
--- a/SivStudies/resources/etls/idr-data.xml
+++ b/SivStudies/resources/etls/idr-data.xml
@@ -37,7 +37,7 @@
                     <setting name="dataSourceSchema" value="bimber_data"/>
                     <setting name="dataSourceQuery" value="idrSampleSource"/>
                     <setting name="dataSourceSubjectColumn" value="Id"/>
-                    <setting name="dataSourceColumns" value="Id,date,sampleId,sampleType,quantity"/>
+                    <setting name="dataSourceColumns" value="Id,date,sampleId,sampleType,quantity,freezer,rack,box,position"/>
                     <setting name="dataSourceColumnDefaults" value="dataSource=Hansen/IDR"/>
 
                     <setting name="targetSchema" value="study"/>
diff --git a/SivStudies/resources/queries/study/samples.query.xml b/SivStudies/resources/queries/study/samples.query.xml
index 3aaacd32..3d5fc433 100644
--- a/SivStudies/resources/queries/study/samples.query.xml
+++ b/SivStudies/resources/queries/study/samples.query.xml
@@ -23,6 +23,18 @@
                     <column columnName="quantity_units">
                         <columnTitle>Quantity Units</columnTitle>
                     </column>
+                    <column columnName="freezer">
+                        <columnTitle>Freezer</columnTitle>
+                    </column>
+                    <column columnName="rack">
+                        <columnTitle>Rack</columnTitle>
+                    </column>
+                    <column columnName="box">
+                        <columnTitle>Box</columnTitle>
+                    </column>
+                    <column columnName="position">
+                        <columnTitle>Position</columnTitle>
+                    </column>
                     <column columnName="comments">
                         <columnTitle>Comments</columnTitle>
                         <inputType>textarea</inputType>
diff --git a/SivStudies/resources/queries/study/samples/.qview.xml b/SivStudies/resources/queries/study/samples/.qview.xml
index 0e70ce34..31300da7 100644
--- a/SivStudies/resources/queries/study/samples/.qview.xml
+++ b/SivStudies/resources/queries/study/samples/.qview.xml
@@ -7,6 +7,10 @@
         <column name="quantity"/>
         <column name="quantity_units"/>
         <column name="sampleId"/>
+        <column name="freezer"/>
+        <column name="rack"/>
+        <column name="box"/>
+        <column name="position"/>
         <column name="comments"/>
         <column name="dataSource"/>
         <column name="viralLoad"/>
diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
index 18199462..f6adf7cf 100644
--- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
+++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
@@ -372,6 +372,18 @@
             <column columnName="quantity_units">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="freezer">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="rack">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="box">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="position">
+                <datatype>varchar</datatype>
+            </column>
             <column columnName="comments">
                 <datatype>varchar</datatype>
             </column>

From 9d41688487b47695d847680c56d66da098df27b0 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 18 Sep 2025 12:46:56 -0700
Subject: [PATCH 10/40] Add enddate to pvl_outcomes

---
 SivStudies/resources/queries/study/pvl_outcomes.query.xml   | 6 +++++-
 SivStudies/resources/queries/study/pvl_outcomes/.qview.xml  | 1 +
 .../referenceStudy/study/datasets/datasets_metadata.xml     | 3 +++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/SivStudies/resources/queries/study/pvl_outcomes.query.xml b/SivStudies/resources/queries/study/pvl_outcomes.query.xml
index e0603880..c61fefb7 100644
--- a/SivStudies/resources/queries/study/pvl_outcomes.query.xml
+++ b/SivStudies/resources/queries/study/pvl_outcomes.query.xml
@@ -5,7 +5,11 @@
                 <columns>
                     <column columnName="Id"/>
                     <column columnName="date">
-                        <columnTitle>Date</columnTitle>
+                        <columnTitle>Window Start</columnTitle>
+                        <formatString>Date</formatString>
+                    </column>
+                    <column columnName="enddate">
+                        <columnTitle>Window End</columnTitle>
                         <formatString>Date</formatString>
                     </column>
                     <column columnName="outcome">
diff --git a/SivStudies/resources/queries/study/pvl_outcomes/.qview.xml b/SivStudies/resources/queries/study/pvl_outcomes/.qview.xml
index 05ad5dcf..8be5fbb7 100644
--- a/SivStudies/resources/queries/study/pvl_outcomes/.qview.xml
+++ b/SivStudies/resources/queries/study/pvl_outcomes/.qview.xml
@@ -2,6 +2,7 @@
     <columns>
         <column name="Id"/>
         <column name="date"/>
+        <column name="enddate"/>
         <column name="outcome"/>
         <column name="numeric_value"/>
         <column name="string_value"/>
diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
index f6adf7cf..84867e79 100644
--- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
+++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
@@ -563,6 +563,9 @@
                 <datatype>timestamp</datatype>
                 <conceptURI>http://cpas.labkey.com/laboratory#sampleDate</conceptURI>
             </column>
+            <column columnName="enddate">
+                <datatype>timestamp</datatype>
+            </column>
             <column columnName="dataSource">
                 <datatype>varchar</datatype>
             </column>

From 372b562912aab3eed96232e49c27e90d86880ecc Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 18 Sep 2025 13:36:12 -0700
Subject: [PATCH 11/40] Remove duplicate IDs in AddMissingIdrSubjects

---
 .../src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java b/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java
index c0873e31..2aaa4407 100644
--- a/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java
+++ b/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java
@@ -2,6 +2,7 @@
 
 import org.apache.xmlbeans.XmlException;
 import org.jetbrains.annotations.NotNull;
+import org.labkey.api.collections.CaseInsensitiveHashSet;
 import org.labkey.api.data.Container;
 import org.labkey.api.data.ContainerManager;
 import org.labkey.api.data.TableSelector;
@@ -49,12 +50,13 @@ public RecordedActionSet run(@NotNull PipelineJob pipelineJob) throws PipelineJo
         List<String> allIds = new ArrayList<>(new TableSelector(us2.getTable("subjects"), PageFlowUtil.set("Rh"), null, null).getArrayList(String.class));
 
         allIds.removeAll(existingIds);
-
         if (allIds.isEmpty())
         {
             return null;
         }
 
+        allIds = new ArrayList<>(new CaseInsensitiveHashSet(allIds));
+
         pipelineJob.getLogger().info("Creating {} subjects", allIds.size());
         List<Map<String, Object>> toInsert = new ArrayList<>();
         allIds.forEach(id -> {

From d2befb6908d7a95f3586620e7c0197610850e4ee Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 18 Sep 2025 13:36:52 -0700
Subject: [PATCH 12/40] Bugfix to AddMissingIdrSubjects

---
 .../src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java b/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java
index 2aaa4407..fb8f9de4 100644
--- a/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java
+++ b/SivStudies/src/org/labkey/sivstudies/etl/AddMissingIdrSubjects.java
@@ -52,7 +52,7 @@ public RecordedActionSet run(@NotNull PipelineJob pipelineJob) throws PipelineJo
         allIds.removeAll(existingIds);
         if (allIds.isEmpty())
         {
-            return null;
+            return new RecordedActionSet();
         }
 
         allIds = new ArrayList<>(new CaseInsensitiveHashSet(allIds));
@@ -77,7 +77,7 @@ public RecordedActionSet run(@NotNull PipelineJob pipelineJob) throws PipelineJo
             throw new PipelineJobException(e);
         }
 
-        return null;
+        return new RecordedActionSet();
     }
 
     @Override

From 7dc523c92df4f4e74a35a0d0764d2dc5b739e48f Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 19 Sep 2025 10:01:05 -0700
Subject: [PATCH 13/40] Expand study triggers and update cohort fields

---
 SivStudies/resources/etls/idr-data.xml        |   2 +-
 .../queries/study/assignment.query.xml        |  13 +-
 .../queries/study/assignment/.qview.xml       |   3 +-
 .../study/datasets/datasets_metadata.xml      |   5 +-
 .../query/DefaultDatasetTrigger.java          |  87 +++++++++++++
 .../query/NumericValuesTrigger.java           | 119 ++++++++++++++++++
 .../query/SivStudiesCustomizer.java           |  21 ++++
 7 files changed, 245 insertions(+), 5 deletions(-)
 create mode 100644 SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java
 create mode 100644 SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java

diff --git a/SivStudies/resources/etls/idr-data.xml b/SivStudies/resources/etls/idr-data.xml
index 066afee5..909ee84e 100644
--- a/SivStudies/resources/etls/idr-data.xml
+++ b/SivStudies/resources/etls/idr-data.xml
@@ -15,7 +15,7 @@
                     <setting name="dataSourceQuery" value="subjects"/>
                     <setting name="dataSourceSubjectColumn" value="Rh"/>
                     <setting name="dataSourceColumns" value="Rh,cohortStart,cohortEnd,cohort,RhCode"/>
-                    <setting name="dataSourceColumnMapping" value="Rh=Id,cohortStart=date,cohortEnd=enddate,RhCode=cohortId,cohort=study"/>
+                    <setting name="dataSourceColumnMapping" value="Rh=Id,cohortStart=date,cohortEnd=enddate,RhCode=cohortAlias,cohort=study"/>
                     <setting name="dataSourceColumnDefaults" value="dataSource=Hansen/IDR"/>
                     <setting name="dataSourceAdditionalFilters" value="cohortStart~neq=0000-00-00"/>
 
diff --git a/SivStudies/resources/queries/study/assignment.query.xml b/SivStudies/resources/queries/study/assignment.query.xml
index ed31de81..85ef6965 100644
--- a/SivStudies/resources/queries/study/assignment.query.xml
+++ b/SivStudies/resources/queries/study/assignment.query.xml
@@ -18,8 +18,8 @@
                     <column columnName="subgroup">
                         <columnTitle>Sub-Group</columnTitle>
                     </column>
-                    <column columnName="cohortId">
-                        <columnTitle>Cohort ID</columnTitle>
+                    <column columnName="cohortAlias">
+                        <columnTitle>Cohort Alias</columnTitle>
                     </column>
                     <column columnName="dataSource">
                         <columnTitle>Data Source</columnTitle>
@@ -27,6 +27,15 @@
                     <column columnName="category">
                         <columnTitle>Category</columnTitle>
                     </column>
+                    <column columnName="cohortId">
+                        <columnTitle>Cohort ID</columnTitle>
+                        <fk>
+                            <fkDbSchema>studies</fkDbSchema>
+                            <fkTable>studyCohorts</fkTable>
+                            <fkColumnName>rowId</fkColumnName>
+                            <fkDisplayColumnName>label</fkDisplayColumnName>
+                        </fk>
+                    </column>
                     <column columnName="description">
                         <columnTitle>Description</columnTitle>
                         <isHidden>true</isHidden>
diff --git a/SivStudies/resources/queries/study/assignment/.qview.xml b/SivStudies/resources/queries/study/assignment/.qview.xml
index b10c793d..d060c788 100644
--- a/SivStudies/resources/queries/study/assignment/.qview.xml
+++ b/SivStudies/resources/queries/study/assignment/.qview.xml
@@ -5,8 +5,9 @@
         <column name="enddate"/>
         <column name="study"/>
         <column name="subgroup"/>
-        <column name="cohortId"/>
+        <column name="cohortAlias"/>
         <column name="category"/>
+        <column name="cohortId"/>
         <column name="dataSource"/>
     </columns>
     <sorts>
diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
index 84867e79..2e2124fe 100644
--- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
+++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
@@ -330,9 +330,12 @@
             <column columnName="subgroup">
                 <datatype>varchar</datatype>
             </column>
-            <column columnName="cohortId">
+            <column columnName="cohortAlias">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="cohortId">
+                <datatype>integer</datatype>
+            </column>
             <column columnName="description">
                 <datatype>varchar</datatype>
             </column>
diff --git a/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java b/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java
new file mode 100644
index 00000000..3d5b21a0
--- /dev/null
+++ b/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java
@@ -0,0 +1,87 @@
+package org.labkey.sivstudies.query;
+
+import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.labkey.api.data.Container;
+import org.labkey.api.data.TableInfo;
+import org.labkey.api.data.triggers.Trigger;
+import org.labkey.api.data.triggers.TriggerFactory;
+import org.labkey.api.query.ValidationException;
+import org.labkey.api.security.User;
+import org.labkey.api.util.logging.LogHelper;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+public class DefaultDatasetTrigger implements Trigger
+{
+    protected static final Logger _log = LogHelper.getLogger(DefaultDatasetTrigger.class, "Messages related to DefaultDatasetTrigger");
+
+    public static class Factory implements TriggerFactory
+    {
+        public Factory()
+        {
+
+        }
+
+        @Override
+        public @NotNull Collection<Trigger> createTrigger(@Nullable Container c, TableInfo table, Map<String, Object> extraContext)
+        {
+            return List.of(new DefaultDatasetTrigger());
+        }
+
+    }
+
+    @Override
+    public void beforeInsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
+    {
+        beforeInsert(table, c, user, newRow, errors, extraContext, null);
+    }
+
+    @Override
+    public void beforeInsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, ValidationException errors, Map<String, Object> extraContext, @Nullable Map<String, Object> existingRecord) throws ValidationException
+    {
+        beforeUpsert(table, c, user, newRow, existingRecord, errors, extraContext);
+    }
+
+    @Override
+    public void beforeUpdate(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, @Nullable Map<String, Object> oldRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
+    {
+        beforeUpsert(table, c, user, newRow, oldRow, errors, extraContext);
+    }
+
+    private void beforeUpsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, @Nullable Map<String, Object> oldRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
+    {
+        if (newRow == null)
+        {
+            _log.error("newRow was null. Unsure when this would ever happen", new Exception());
+            return;
+        }
+
+        // Simplify properties:
+        mergeOldToNewRow(newRow, oldRow);
+
+        doBeforeUpsert(table, c, user, newRow, oldRow, errors, extraContext);
+    }
+
+    protected void doBeforeUpsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, @Nullable Map<String, Object> oldRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
+    {
+        // Allow subclasses to implement code here
+    }
+
+    private void mergeOldToNewRow(@NotNull Map<String, Object> newRow, @Nullable Map<String, Object> oldRow) throws ValidationException
+    {
+        if (oldRow != null)
+        {
+            for (String propName : oldRow.keySet())
+            {
+                if (!newRow.containsKey(propName))
+                {
+                    newRow.put(propName, oldRow.get(propName));
+                }
+            }
+        }
+    }
+}
diff --git a/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java b/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java
new file mode 100644
index 00000000..83f44261
--- /dev/null
+++ b/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java
@@ -0,0 +1,119 @@
+package org.labkey.sivstudies.query;
+
+import org.apache.commons.lang3.math.NumberUtils;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.labkey.api.data.ColumnInfo;
+import org.labkey.api.data.Container;
+import org.labkey.api.data.TableInfo;
+import org.labkey.api.data.triggers.Trigger;
+import org.labkey.api.data.triggers.TriggerFactory;
+import org.labkey.api.query.SimpleValidationError;
+import org.labkey.api.query.ValidationException;
+import org.labkey.api.security.User;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class NumericValuesTrigger extends DefaultDatasetTrigger
+{
+    public static class Factory implements TriggerFactory
+    {
+        // This map allows caller to supply a list of <StringValue> -> <TargetField>. If that string is found in a numeric field, it will be
+        private final List<StringTransformer> _stringTransformers;
+
+        public Factory()
+        {
+            this(null);
+        }
+
+        public Factory(@Nullable List<StringTransformer> stringTransformers)
+        {
+            _stringTransformers = stringTransformers == null ? Collections.emptyList() : stringTransformers;
+        }
+
+        @Override
+        public @NotNull Collection<Trigger> createTrigger(@Nullable Container c, TableInfo table, Map<String, Object> extraContext)
+        {
+            return List.of(new NumericValuesTrigger(_stringTransformers));
+        }
+    }
+
+    private final List<StringTransformer> _stringTransformers;
+
+    public NumericValuesTrigger(List<StringTransformer> stringTransformers)
+    {
+        _stringTransformers = stringTransformers;
+    }
+
+    @Override
+    protected void doBeforeUpsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, @Nullable Map<String, Object> oldRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
+    {
+        inspectNumericValues(table, newRow, errors);
+    }
+
+    private void inspectNumericValues(TableInfo table, Map<String, Object> row, ValidationException errors)
+    {
+        for (String propName : row.keySet())
+        {
+            if (row.get(propName) == null)
+            {
+                continue;
+            }
+
+            ColumnInfo ci = table.getColumn(propName);
+            if (ci == null)
+            {
+                continue;
+            }
+
+            if (!Number.class.isAssignableFrom(ci.getJdbcType().getJavaClass()))
+            {
+                continue;
+            }
+
+            String val = String.valueOf(row.get(propName));
+            if (NumberUtils.isCreatable(val))
+            {
+                return;
+            }
+
+            // commas are a common problem:
+            val = val.replaceAll(",", "");
+            if (NumberUtils.isCreatable(val))
+            {
+                row.put(propName, val);
+                return;
+            }
+
+            // The Rlabkey API sending NAs as strings is another common problem:
+            if ("NA".equalsIgnoreCase(val))
+            {
+                row.put(propName, null);
+                return;
+            }
+
+            for (StringTransformer stringTransformer : _stringTransformers)
+            {
+                stringTransformer.inspectValue(table, row, val, propName, errors);
+                val = String.valueOf(row.get(propName));
+            }
+
+            if (NumberUtils.isCreatable(val))
+            {
+                row.put(propName, val);
+                return;
+            }
+
+            errors.addError(new SimpleValidationError("Non-numeric value for field " + propName + ": " + val, propName, ValidationException.SEVERITY.ERROR));
+        }
+    }
+
+    public interface StringTransformer
+    {
+        // This method allows code to inspect and modify non-numeric values
+        public void inspectValue(TableInfo ti, Map<String, Object> row, String stringValue, String propName, ValidationException errors);
+    }
+}
diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
index 9d82243a..6e59d69a 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
@@ -1,5 +1,6 @@
 package org.labkey.sivstudies.query;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.labkey.api.collections.CaseInsensitiveHashSet;
 import org.labkey.api.data.AbstractTableInfo;
@@ -69,6 +70,7 @@ public void performDatasetCustomization(DatasetTable ds)
 
             appendDemographicsColumns(ati);
 
+            addNumericValuesTrigger(ati);
             if ("viralLoads".equalsIgnoreCase(ds.getName()))
             {
                 customizeViralLoads(ati);
@@ -494,6 +496,25 @@ private BaseColumnInfo getWrappedIdCol(UserSchema targetQueryUserSchema, String
         return col;
     }
 
+    private void addNumericValuesTrigger(AbstractTableInfo ati)
+    {
+        List<NumericValuesTrigger.StringTransformer> stringTransformers = new ArrayList<>();
+        if ("immunizations".equalsIgnoreCase(ati.getName()))
+        {
+            stringTransformers.add((ti, row, stringValue, propName, errors) -> {
+                if ("quantity".equalsIgnoreCase(propName) & "Supernatant".equalsIgnoreCase(stringValue))
+                {
+                    row.put("quantity", null);
+                    String comments = row.get("comments") == null ? null : StringUtils.trimToNull(String.valueOf(row.get("comments")));
+                    comments = (comments == null ? "" : comments + ", ") + "Quantity: Supernatant";
+                    row.put("comments", comments);
+                }
+            });
+        }
+
+        ati.addTriggerFactory(new NumericValuesTrigger.Factory(stringTransformers));
+    }
+
     private void customizeViralLoads(AbstractTableInfo ati)
     {
         ati.addTriggerFactory(new ViralLoadsTriggerFactory());

From 5c54e539f7bf2157f11fcd27e3b5c8b0b9328a92 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 19 Sep 2025 10:26:46 -0700
Subject: [PATCH 14/40] Bugfix to ART fields

---
 SivStudies/resources/views/dataNotes.html                 | 6 ++++++
 SivStudies/resources/views/studiesAdmin.html              | 3 +++
 .../org/labkey/sivstudies/query/SivStudiesCustomizer.java | 8 ++++----
 3 files changed, 13 insertions(+), 4 deletions(-)
 create mode 100644 SivStudies/resources/views/dataNotes.html

diff --git a/SivStudies/resources/views/dataNotes.html b/SivStudies/resources/views/dataNotes.html
new file mode 100644
index 00000000..0f56d300
--- /dev/null
+++ b/SivStudies/resources/views/dataNotes.html
@@ -0,0 +1,6 @@
+This page summarizes
+<ul>
+    <li>The fields to calculate timePostSivChallenge require a record in studies.subjectAnchorDates from the same Id where eventLabel = 'SIV Infection'. There may be records in the treatments table for SIV challenges; these do not count</li>
+    <li>The fields to calculate overlapping PVLs require a record in the viral_load table from the same Id/date, where target = SIV and where sampleType = plasma.</li>
+    <li>The fields to calculate ART-related date require record(s) in the treatments table from the same Id, where category = 'ART'</li>
+</ul>
\ No newline at end of file
diff --git a/SivStudies/resources/views/studiesAdmin.html b/SivStudies/resources/views/studiesAdmin.html
index fb5e9102..8cebaf45 100644
--- a/SivStudies/resources/views/studiesAdmin.html
+++ b/SivStudies/resources/views/studiesAdmin.html
@@ -27,6 +27,9 @@
                 },{
                     name: 'Notification Admin',
                     url: LABKEY.ActionURL.buildURL('ldk', 'notificationAdmin.view')
+                },{
+                    name: 'Notes For Managing Data',
+                    url: LABKEY.ActionURL.buildURL('sivstudies', 'dataNotes.view')
                 }]
             }]
         }]
diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
index 6e59d69a..63293efa 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
@@ -427,13 +427,13 @@ public TableInfo getLookupTableInfo()
                 UserSchema targetSchema = ds.getUserSchema().getDefaultSchema().getUserSchema(targetSchemaName);
                 QueryDefinition qd = QueryService.get().createQueryDef(u, targetSchemaContainer, targetSchema, name);
                 qd.setSql("SELECT\n" +
-                        "max(tr.date) as artInitiation,\n" +
-                        "CONVERT(TIMESTAMPDIFF('SQL_TSI_DAY', CAST(max(tr.date) AS DATE), CAST(c." + dateColName + " AS DATE)), INTEGER) as daysPostArtInitiation,\n" +
-                        "CONVERT(age_in_months(CAST(max(tr.date) AS DATE), CAST(c." + dateColName + " AS DATE)), FLOAT) as monthsPostArtInitiation,\n" +
+                        "min(tr.date) as artInitiation,\n" +
+                        "CONVERT(TIMESTAMPDIFF('SQL_TSI_DAY', CAST(min(tr.date) AS DATE), CAST(c." + dateColName + " AS DATE)), INTEGER) as daysPostArtInitiation,\n" +
+                        "CONVERT(age_in_months(CAST(min(tr.date) AS DATE), CAST(c." + dateColName + " AS DATE)), FLOAT) as monthsPostArtInitiation,\n" +
                         "max(tr.enddate) as artRelease,\n" +
                         "CONVERT(CASE WHEN max(tr.enddate) IS NULL THEN NULL ELSE TIMESTAMPDIFF('SQL_TSI_DAY', CAST(max(tr.enddate) AS DATE), CAST(c." + dateColName + " AS DATE)) END, INTEGER) as daysPostArtRelease,\n" +
                         "CONVERT(CASE WHEN max(tr.enddate) IS NULL THEN NULL ELSE age_in_months(CAST(max(tr.enddate) AS DATE), CAST(c." + dateColName + " AS DATE)) END, FLOAT) as monthsPostArtRelease,\n" +
-                        "CAST(CASE WHEN CAST(max(tr.date) AS DATE) < CAST(c." + dateCol.getFieldKey().toString()  + " AS DATE) AND CAST(max(coalesce(tr.enddate, now())) AS DATE) >= CAST(c." + dateCol.getFieldKey().toString() + " AS DATE) THEN 'Y' ELSE null END as VARCHAR) as onArt,\n" +
+                        "CAST(CASE WHEN CAST(min(tr.date) AS DATE) <= CAST(c." + dateCol.getFieldKey().toString()  + " AS DATE) AND CAST(max(coalesce(tr.enddate, now())) AS DATE) >= CAST(c." + dateCol.getFieldKey().toString() + " AS DATE) THEN 'Y' ELSE null END as VARCHAR) as onArt,\n" +
                         "GROUP_CONCAT(DISTINCT tr.treatment) AS artTreatment,\n" +
                         "c." + pkCol.getFieldKey().toString() + "\n" +
                         "FROM \"" + schemaName + "\".\"" + queryName + "\" c " +

From 8c1710710e4956990bffe0ea7b28ead192889362 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 19 Sep 2025 11:03:35 -0700
Subject: [PATCH 15/40] Bugfix to NumericValuesTrigger

---
 .../org/labkey/sivstudies/query/NumericValuesTrigger.java    | 2 +-
 .../org/labkey/sivstudies/query/SivStudiesCustomizer.java    | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java b/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java
index 83f44261..376bd7a2 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java
@@ -101,7 +101,7 @@ private void inspectNumericValues(TableInfo table, Map<String, Object> row, Vali
                 val = String.valueOf(row.get(propName));
             }
 
-            if (NumberUtils.isCreatable(val))
+            if (val == null || NumberUtils.isCreatable(val))
             {
                 row.put(propName, val);
                 return;
diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
index 63293efa..03d75117 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
@@ -75,6 +75,11 @@ public void performDatasetCustomization(DatasetTable ds)
             {
                 customizeViralLoads(ati);
             }
+
+            if ("assignment".equalsIgnoreCase(ds.getName()))
+            {
+                ati.addTriggerFactory(StudiesService.get().getStudiesTriggerFactory());
+            }
         }
         else
         {

From 3d826242993c0f49db252ec481574462c3117f0e Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 19 Sep 2025 14:05:46 -0700
Subject: [PATCH 16/40] Clean up trigger/customizer layer code

---
 SivStudies/resources/queries/study/studyData.query.xml          | 1 +
 .../src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/SivStudies/resources/queries/study/studyData.query.xml b/SivStudies/resources/queries/study/studyData.query.xml
index e78b0c84..ab97c463 100644
--- a/SivStudies/resources/queries/study/studyData.query.xml
+++ b/SivStudies/resources/queries/study/studyData.query.xml
@@ -7,6 +7,7 @@
                     <column columnName="Id">
                         <conceptURI>http://cpas.labkey.com/Study#ParticipantId</conceptURI>
                         <url>laboratory/dataBrowser.view?subjectId=${Id}</url>
+                        <facetingBehavior>ALWAYS_OFF</facetingBehavior>
                     </column>
                     <column columnName="date">
                         <columnTitle>Date</columnTitle>
diff --git a/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java b/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java
index 3d5b21a0..26efb191 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java
@@ -77,7 +77,7 @@ private void mergeOldToNewRow(@NotNull Map<String, Object> newRow, @Nullable Map
         {
             for (String propName : oldRow.keySet())
             {
-                if (!newRow.containsKey(propName))
+                if (!newRow.containsKey(propName) & oldRow.get(propName) != null)
                 {
                     newRow.put(propName, oldRow.get(propName));
                 }

From 3239871378ac285ddc2ffb0adea85bcd8dd51284 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 19 Sep 2025 14:29:06 -0700
Subject: [PATCH 17/40] Clean up trigger/customizer layer code

---
 .../org/labkey/sivstudies/query/NumericValuesTrigger.java   | 6 +++---
 .../org/labkey/sivstudies/query/SivStudiesCustomizer.java   | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java b/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java
index 376bd7a2..38eac57b 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/NumericValuesTrigger.java
@@ -74,7 +74,7 @@ private void inspectNumericValues(TableInfo table, Map<String, Object> row, Vali
                 continue;
             }
 
-            String val = String.valueOf(row.get(propName));
+            String val = row.get(propName) == null ? null : String.valueOf(row.get(propName));
             if (NumberUtils.isCreatable(val))
             {
                 return;
@@ -89,7 +89,7 @@ private void inspectNumericValues(TableInfo table, Map<String, Object> row, Vali
             }
 
             // The Rlabkey API sending NAs as strings is another common problem:
-            if ("NA".equalsIgnoreCase(val))
+            if ("NA".equalsIgnoreCase(val) || "null".equalsIgnoreCase(val))
             {
                 row.put(propName, null);
                 return;
@@ -98,7 +98,7 @@ private void inspectNumericValues(TableInfo table, Map<String, Object> row, Vali
             for (StringTransformer stringTransformer : _stringTransformers)
             {
                 stringTransformer.inspectValue(table, row, val, propName, errors);
-                val = String.valueOf(row.get(propName));
+                val = row.get(propName) == null ? null : String.valueOf(row.get(propName));
             }
 
             if (val == null || NumberUtils.isCreatable(val))
diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
index 03d75117..69a0174e 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
@@ -507,7 +507,7 @@ private void addNumericValuesTrigger(AbstractTableInfo ati)
         if ("immunizations".equalsIgnoreCase(ati.getName()))
         {
             stringTransformers.add((ti, row, stringValue, propName, errors) -> {
-                if ("quantity".equalsIgnoreCase(propName) & "Supernatant".equalsIgnoreCase(stringValue))
+                if ("quantity".equalsIgnoreCase(propName) & ("Supernatant".equalsIgnoreCase(stringValue) | "Supernatent".equalsIgnoreCase(stringValue)))
                 {
                     row.put("quantity", null);
                     String comments = row.get("comments") == null ? null : StringUtils.trimToNull(String.valueOf(row.get("comments")));

From 424c665b88d546bec2bc48086767d15896b2d6da Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 19 Sep 2025 14:46:36 -0700
Subject: [PATCH 18/40] Create fields to coalesce name/label for studies

---
 SivStudies/resources/queries/study/assignment.query.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/SivStudies/resources/queries/study/assignment.query.xml b/SivStudies/resources/queries/study/assignment.query.xml
index 85ef6965..00df5943 100644
--- a/SivStudies/resources/queries/study/assignment.query.xml
+++ b/SivStudies/resources/queries/study/assignment.query.xml
@@ -33,7 +33,7 @@
                             <fkDbSchema>studies</fkDbSchema>
                             <fkTable>studyCohorts</fkTable>
                             <fkColumnName>rowId</fkColumnName>
-                            <fkDisplayColumnName>label</fkDisplayColumnName>
+                            <fkDisplayColumnName>labelOrName</fkDisplayColumnName>
                         </fk>
                     </column>
                     <column columnName="description">

From db5d771ecc846bfa03fd6228df87c99d625b1184 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 19 Sep 2025 15:22:33 -0700
Subject: [PATCH 19/40] Expand SIV studies notification

---
 .../study/pvlWithoutInfectionDate.query.xml   |  9 ++++++
 .../queries/study/pvlWithoutInfectionDate.sql | 10 +++++++
 .../SivStudiesDataValidationNotification.java | 29 +++++++++----------
 3 files changed, 32 insertions(+), 16 deletions(-)
 create mode 100644 SivStudies/resources/queries/study/pvlWithoutInfectionDate.query.xml
 create mode 100644 SivStudies/resources/queries/study/pvlWithoutInfectionDate.sql

diff --git a/SivStudies/resources/queries/study/pvlWithoutInfectionDate.query.xml b/SivStudies/resources/queries/study/pvlWithoutInfectionDate.query.xml
new file mode 100644
index 00000000..bd636239
--- /dev/null
+++ b/SivStudies/resources/queries/study/pvlWithoutInfectionDate.query.xml
@@ -0,0 +1,9 @@
+<query xmlns="http://labkey.org/data/xml/query">
+    <metadata>
+        <tables xmlns="http://labkey.org/data/xml">
+            <table tableName="pvlWithoutInfectionDate" tableDbType="NOT_IN_DB">
+                <tableTitle>Animals With PVL Data But No Infection Date</tableTitle>
+            </table>
+        </tables>
+    </metadata>
+</query>
diff --git a/SivStudies/resources/queries/study/pvlWithoutInfectionDate.sql b/SivStudies/resources/queries/study/pvlWithoutInfectionDate.sql
new file mode 100644
index 00000000..1a655cbd
--- /dev/null
+++ b/SivStudies/resources/queries/study/pvlWithoutInfectionDate.sql
@@ -0,0 +1,10 @@
+SELECT
+    vl.Id,
+    max(vl.result) as maxViralLoad
+
+FROM study.viralloads vl
+WHERE
+    vl.result IS NOT NULL AND
+    ((vl.lod is not NULL AND vl.result > vl.lod) OR (vl.lod IS NULL AND vl.result > 50)) AND
+    vl.timePostSivChallenge.infectionDate IS NULL
+GROUP BY vl.Id
\ No newline at end of file
diff --git a/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java b/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java
index 314c662a..faeae93c 100644
--- a/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java
+++ b/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java
@@ -63,6 +63,8 @@ public String getEmailSubject(Container c)
         Date now = new Date();
 
         duplicateInfectionCheck(c, u, msg);
+        infectionAnchorDateDiscordance(c, u, msg);
+        pvlWithoutInfectionDate(c, u, msg);
 
         if (!msg.isEmpty())
         {
@@ -91,35 +93,30 @@ private TableInfo getTableInfo(User u, Container c, String schemaName, String qu
 
     private void duplicateInfectionCheck(Container c, User u, StringBuilder msg)
     {
-        String schemaName = "study";
-        String queryName = "duplicateInfectionDates";
-
-        TableInfo ti = getTableInfo(u, c, schemaName, queryName);
-
-        TableSelector ts = new TableSelector(ti);
-        long count = ts.getRowCount();
-        if (count > 0)
-        {
-            msg.append("<b>WARNING: There are ").append(count).append(" duplicate infection date records</b><br>\n");
-            msg.append("<p><a href='").append(getExecuteQueryUrl(c, schemaName, queryName, null)).append("'>Click here to view them</a><br>\n\n");
-            msg.append("<hr>\n\n");
-        }
+        genericQueryCheck(c, u, msg, "study", "duplicateInfectionDates", "duplicate infection date records");
     }
 
     private void infectionAnchorDateDiscordance(Container c, User u, StringBuilder msg)
     {
-        String schemaName = "study";
-        String queryName = "infectionAnchorDateDiscordance";
+        genericQueryCheck(c, u, msg, "study", "infectionAnchorDateDiscordance", "records with discordant treatment and anchor date SIV infection records");
+    }
 
+    private void genericQueryCheck(Container c, User u, StringBuilder msg, String schemaName, String queryName, String message)
+    {
         TableInfo ti = getTableInfo(u, c, schemaName, queryName);
 
         TableSelector ts = new TableSelector(ti);
         long count = ts.getRowCount();
         if (count > 0)
         {
-            msg.append("<b>WARNING: There are ").append(count).append(" records with discordant treatment and anchor date SIV infection records</b><br>\n");
+            msg.append("<b>WARNING: There are ").append(count).append(" " + message + "</b><br>\n");
             msg.append("<p><a href='").append(getExecuteQueryUrl(c, schemaName, queryName, null)).append("'>Click here to view them</a><br>\n\n");
             msg.append("<hr>\n\n");
         }
     }
+
+    private void pvlWithoutInfectionDate(Container c, User u, StringBuilder msg)
+    {
+        genericQueryCheck(c, u, msg, "study", "pvlWithoutInfectionDate", "animals with PVL data but no record of SIV infection");
+    }
 }

From 6d8c507311c98070bfa1c3941b68effbb20a7533 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sat, 20 Sep 2025 07:29:09 -0700
Subject: [PATCH 20/40] Dont add NumericValuesTrigger to PVL table

---
 .../org/labkey/sivstudies/query/SivStudiesCustomizer.java   | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
index 69a0174e..4a8f3e36 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
@@ -503,6 +503,12 @@ private BaseColumnInfo getWrappedIdCol(UserSchema targetQueryUserSchema, String
 
     private void addNumericValuesTrigger(AbstractTableInfo ati)
     {
+        // This behavior conflicts with ViralLoadsTriggerFactory
+        if ("viralLoads".equalsIgnoreCase(ati.getName()))
+        {
+            return;
+        }
+
         List<NumericValuesTrigger.StringTransformer> stringTransformers = new ArrayList<>();
         if ("immunizations".equalsIgnoreCase(ati.getName()))
         {

From bfabbd2a82f4f8088413501162b71929de761053 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sat, 27 Sep 2025 09:06:11 -0700
Subject: [PATCH 21/40] Add QCLabel filter

---
 PMR/resources/etls/prime-chemistryResults.xml   | 3 +++
 PMR/resources/etls/prime-clinpathRuns.xml       | 3 +++
 PMR/resources/etls/prime-hematologyResults.xml  | 3 +++
 PMR/resources/etls/prime-histology.xml          | 3 +++
 PMR/resources/etls/prime-microbiology.xml       | 3 +++
 PMR/resources/etls/prime-pathologyDiagnoses.xml | 3 +++
 6 files changed, 18 insertions(+)

diff --git a/PMR/resources/etls/prime-chemistryResults.xml b/PMR/resources/etls/prime-chemistryResults.xml
index 0c0cd1c9..dc8666de 100644
--- a/PMR/resources/etls/prime-chemistryResults.xml
+++ b/PMR/resources/etls/prime-chemistryResults.xml
@@ -18,6 +18,9 @@
                     <column>modified</column>
                     <column>QCState/Label</column>
                 </sourceColumns>
+                <sourceFilters>
+                    <sourceFilter column="qcstate/label" operator="eq" value="Completed"/>
+                </sourceFilters>
             </source>
             <destination schemaName="study" queryName="chemistryResults" targetOption="merge" bulkLoad="true" batchSize="1000">
                 <alternateKeys>
diff --git a/PMR/resources/etls/prime-clinpathRuns.xml b/PMR/resources/etls/prime-clinpathRuns.xml
index f6afe0b2..42af3a09 100644
--- a/PMR/resources/etls/prime-clinpathRuns.xml
+++ b/PMR/resources/etls/prime-clinpathRuns.xml
@@ -19,6 +19,9 @@
                     <column>modified</column>
                     <column>QCState/Label</column>
                 </sourceColumns>
+                <sourceFilters>
+                    <sourceFilter column="qcstate/label" operator="eq" value="Completed"/>
+                </sourceFilters>
             </source>
             <destination schemaName="study" queryName="clinpathRuns" bulkLoad="true" targetOption="merge" batchSize="1000">
                 <alternateKeys>
diff --git a/PMR/resources/etls/prime-hematologyResults.xml b/PMR/resources/etls/prime-hematologyResults.xml
index 9df7326e..510f69be 100644
--- a/PMR/resources/etls/prime-hematologyResults.xml
+++ b/PMR/resources/etls/prime-hematologyResults.xml
@@ -18,6 +18,9 @@
                     <column>modified</column>
                     <column>QCState/Label</column>
                 </sourceColumns>
+                <sourceFilters>
+                    <sourceFilter column="qcstate/label" operator="eq" value="Completed"/>
+                </sourceFilters>
             </source>
             <destination schemaName="study" queryName="hematologyResults" targetOption="merge" bulkLoad="true" batchSize="1000">
                 <alternateKeys>
diff --git a/PMR/resources/etls/prime-histology.xml b/PMR/resources/etls/prime-histology.xml
index 27a21350..9f59b0b3 100644
--- a/PMR/resources/etls/prime-histology.xml
+++ b/PMR/resources/etls/prime-histology.xml
@@ -17,6 +17,9 @@
                     <column>modified</column>
                     <column>QCState/Label</column>
                 </sourceColumns>
+                <sourceFilters>
+                    <sourceFilter column="qcstate/label" operator="eq" value="Completed"/>
+                </sourceFilters>
             </source>
             <destination schemaName="study" queryName="histology" targetOption="merge" bulkLoad="true" batchSize="1000">
                 <alternateKeys>
diff --git a/PMR/resources/etls/prime-microbiology.xml b/PMR/resources/etls/prime-microbiology.xml
index 194c4d83..d49d5158 100644
--- a/PMR/resources/etls/prime-microbiology.xml
+++ b/PMR/resources/etls/prime-microbiology.xml
@@ -18,6 +18,9 @@
                     <column>modified</column>
                     <column>QCState/Label</column>
                 </sourceColumns>
+                <sourceFilters>
+                    <sourceFilter column="qcstate/label" operator="eq" value="Completed"/>
+                </sourceFilters>
             </source>
             <destination schemaName="study" queryName="microbiology" targetOption="merge" bulkLoad="true" batchSize="1000">
                 <alternateKeys>
diff --git a/PMR/resources/etls/prime-pathologyDiagnoses.xml b/PMR/resources/etls/prime-pathologyDiagnoses.xml
index 428ef6e3..172297dc 100644
--- a/PMR/resources/etls/prime-pathologyDiagnoses.xml
+++ b/PMR/resources/etls/prime-pathologyDiagnoses.xml
@@ -16,6 +16,9 @@
                     <column>modified</column>
                     <column>QCState/Label</column>
                 </sourceColumns>
+                <sourceFilters>
+                    <sourceFilter column="qcstate/label" operator="eq" value="Completed"/>
+                </sourceFilters>
             </source>
             <destination schemaName="study" queryName="pathologyDiagnoses" targetOption="merge" bulkLoad="true" batchSize="1000">
                 <alternateKeys>

From 8564e48a32af576f039f20a1f49400485fd60450 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 29 Sep 2025 11:19:36 -0700
Subject: [PATCH 22/40] Add field for number of PVL datapoints

---
 SivStudies/resources/queries/study/pvl_outcomes.query.xml     | 4 ++++
 .../referenceStudy/study/datasets/datasets_metadata.xml       | 3 +++
 2 files changed, 7 insertions(+)

diff --git a/SivStudies/resources/queries/study/pvl_outcomes.query.xml b/SivStudies/resources/queries/study/pvl_outcomes.query.xml
index c61fefb7..53f67bff 100644
--- a/SivStudies/resources/queries/study/pvl_outcomes.query.xml
+++ b/SivStudies/resources/queries/study/pvl_outcomes.query.xml
@@ -21,6 +21,10 @@
                     <column columnName="string_value">
                         <columnTitle>String Value</columnTitle>
                     </column>
+                    <column columnName="numDatapoints">
+                        <columnTitle># Datapoints</columnTitle>
+                        <description>The number of PVL datapoints used for this calculation</description>
+                    </column>
                     <column columnName="comments">
                         <columnTitle>Comments</columnTitle>
                     </column>
diff --git a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
index 2e2124fe..802e195f 100644
--- a/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
+++ b/SivStudies/resources/referenceStudy/study/datasets/datasets_metadata.xml
@@ -585,6 +585,9 @@
             <column columnName="string_value">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="numDatapoints">
+                <datatype>integer</datatype>
+            </column>
             <column columnName="comments">
                 <datatype>varchar</datatype>
             </column>

From 753cd1e6a0c94d7eb9168e8ae81144c4c2439cb3 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 29 Sep 2025 20:25:08 -0700
Subject: [PATCH 23/40] Increase batch size

---
 .../src/org/labkey/sivstudies/etl/SubjectScopedSelect.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java
index 6c750b8b..37b54fa8 100644
--- a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java
+++ b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java
@@ -101,7 +101,7 @@ public boolean isRequired()
         }
     }
 
-    final int BATCH_SIZE = 250;
+    final int BATCH_SIZE = 500;
 
     private MODE getMode()
     {

From 5c8c616ed79e80b4d06e558a1c8bd56814cc6deb Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sat, 4 Oct 2025 07:44:12 -0700
Subject: [PATCH 24/40] Build short delay into github triggers to aid
 cross-repo commits

---
 .github/workflows/build.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index fff5cbff..535315bc 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -10,6 +10,11 @@ jobs:
     if: github.repository == 'BimberLabInternal/BimberLabKeyModules'
     runs-on: ubuntu-latest
     steps:
+      # Note: use slight delay in case there are associated commits across repos
+      - name: "Sleep for 30 seconds"
+        run: sleep 30s
+        shell: bash
+
       - name: "Build DISCVR"
         uses: bimberlabinternal/DevOps/githubActions/discvr-build@master
         with:

From 23761ec7445f98b8c6db19e69ab9e99534b25ee1 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 6 Oct 2025 08:03:12 -0700
Subject: [PATCH 25/40] Switch ETLs to log row count discrepancies

---
 PMR/resources/etls/prime-blooddraws.xml         |  2 ++
 PMR/resources/etls/prime-chemistryResults.xml   |  2 ++
 PMR/resources/etls/prime-clinpathRuns.xml       |  2 ++
 PMR/resources/etls/prime-hematologyResults.xml  |  2 ++
 PMR/resources/etls/prime-histology.xml          |  2 ++
 PMR/resources/etls/prime-microbiology.xml       |  2 ++
 PMR/resources/etls/prime-pathologyDiagnoses.xml |  2 ++
 PMR/resources/etls/prime-weight.xml             |  2 ++
 .../org/labkey/primeseq/etl/VerifyRowCount.java | 17 +++++++++++++++--
 9 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/PMR/resources/etls/prime-blooddraws.xml b/PMR/resources/etls/prime-blooddraws.xml
index 13abeaa1..93d4a6cd 100644
--- a/PMR/resources/etls/prime-blooddraws.xml
+++ b/PMR/resources/etls/prime-blooddraws.xml
@@ -34,6 +34,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="blood"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-chemistryResults.xml b/PMR/resources/etls/prime-chemistryResults.xml
index dc8666de..380c8879 100644
--- a/PMR/resources/etls/prime-chemistryResults.xml
+++ b/PMR/resources/etls/prime-chemistryResults.xml
@@ -40,6 +40,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="chemistryResults"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-clinpathRuns.xml b/PMR/resources/etls/prime-clinpathRuns.xml
index 42af3a09..70b9c6f0 100644
--- a/PMR/resources/etls/prime-clinpathRuns.xml
+++ b/PMR/resources/etls/prime-clinpathRuns.xml
@@ -41,6 +41,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="clinpathRuns"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-hematologyResults.xml b/PMR/resources/etls/prime-hematologyResults.xml
index 510f69be..53d2bec5 100644
--- a/PMR/resources/etls/prime-hematologyResults.xml
+++ b/PMR/resources/etls/prime-hematologyResults.xml
@@ -40,6 +40,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="hematologyResults"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-histology.xml b/PMR/resources/etls/prime-histology.xml
index 9f59b0b3..b20efed0 100644
--- a/PMR/resources/etls/prime-histology.xml
+++ b/PMR/resources/etls/prime-histology.xml
@@ -39,6 +39,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="histology"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-microbiology.xml b/PMR/resources/etls/prime-microbiology.xml
index d49d5158..0f7a74d7 100644
--- a/PMR/resources/etls/prime-microbiology.xml
+++ b/PMR/resources/etls/prime-microbiology.xml
@@ -40,6 +40,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="microbiology"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-pathologyDiagnoses.xml b/PMR/resources/etls/prime-pathologyDiagnoses.xml
index 172297dc..aa9a38a6 100644
--- a/PMR/resources/etls/prime-pathologyDiagnoses.xml
+++ b/PMR/resources/etls/prime-pathologyDiagnoses.xml
@@ -38,6 +38,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="pathologyDiagnoses"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-weight.xml b/PMR/resources/etls/prime-weight.xml
index 435679e8..e08c00c1 100644
--- a/PMR/resources/etls/prime-weight.xml
+++ b/PMR/resources/etls/prime-weight.xml
@@ -37,6 +37,8 @@
                     <setting name="destSchema" value="study"/>
                     <setting name="destQuery" value="weight"/>
                     <setting name="destColumn" value="objectId"/>
+
+                    <setting name="reportOnly" value="true"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java b/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java
index 95d0c2f2..b1dcd860 100644
--- a/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java
+++ b/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java
@@ -1,6 +1,7 @@
 package org.labkey.primeseq.etl;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
 import org.labkey.api.collections.CaseInsensitiveHashMap;
@@ -50,7 +51,8 @@ private enum Settings
         destSchema(true),
         destQuery(true),
         destColumn(true),
-        destAdditionalFilters(false);
+        destAdditionalFilters(false),
+        reportOnly(false);
 
         private final boolean _isRequired;
 
@@ -106,6 +108,11 @@ public void setSettings(Map<String, String> settings)
         _settings.putAll(settings);
     }
 
+    private boolean isReportOnly()
+    {
+        return _settings.containsKey(Settings.reportOnly.name()) && Boolean.parseBoolean(_settings.get(Settings.reportOnly.name()));
+    }
+
     private DataIntegrationService.RemoteConnection getRemoteDataSource(String name, Container c, Logger log) throws IllegalStateException
     {
         DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(name, c, log);
@@ -259,7 +266,13 @@ private void verifyRows(PipelineJob job) throws PipelineJobException
 
         if (source != dest)
         {
-            job.getLogger().error("Row counts do not match (source: {}, dest: {})!", source, dest);
+            if (isReportOnly()) {
+                job.getLogger().info("Row counts do not match (source: {}, dest: {})!", source, dest);
+            }
+            else
+            {
+                job.getLogger().error("Row counts do not match (source: {}, dest: {})!", source, dest);
+            }
         }
     }
 }

From 3c16a6e4e8b221bf07e66907aaddf3fa29329978 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 6 Oct 2025 12:42:24 -0700
Subject: [PATCH 26/40] Drop unused fields

---
 PMR/resources/etls/prime-chemistryResults.xml  | 1 -
 PMR/resources/etls/prime-hematologyResults.xml | 1 -
 2 files changed, 2 deletions(-)

diff --git a/PMR/resources/etls/prime-chemistryResults.xml b/PMR/resources/etls/prime-chemistryResults.xml
index 380c8879..bda1efe8 100644
--- a/PMR/resources/etls/prime-chemistryResults.xml
+++ b/PMR/resources/etls/prime-chemistryResults.xml
@@ -8,7 +8,6 @@
                 <sourceColumns>
                     <column>Id</column>
                     <column>date</column>
-                    <column>ageAtTime</column>
                     <column>testId</column>
                     <column>result</column>
                     <column>units</column>
diff --git a/PMR/resources/etls/prime-hematologyResults.xml b/PMR/resources/etls/prime-hematologyResults.xml
index 53d2bec5..e3aca8f3 100644
--- a/PMR/resources/etls/prime-hematologyResults.xml
+++ b/PMR/resources/etls/prime-hematologyResults.xml
@@ -8,7 +8,6 @@
                 <sourceColumns>
                     <column>Id</column>
                     <column>date</column>
-                    <column>ageAtTime</column>
                     <column>testId</column>
                     <column>result</column>
                     <column>units</column>

From c2f37222dae672562e0b2cda2994f267b60c9adb Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Tue, 7 Oct 2025 10:28:22 -0700
Subject: [PATCH 27/40] Restore tighter validation

---
 PMR/resources/etls/prime-blooddraws.xml         | 2 +-
 PMR/resources/etls/prime-chemistryResults.xml   | 2 +-
 PMR/resources/etls/prime-clinpathRuns.xml       | 2 +-
 PMR/resources/etls/prime-hematologyResults.xml  | 2 +-
 PMR/resources/etls/prime-histology.xml          | 2 +-
 PMR/resources/etls/prime-microbiology.xml       | 2 +-
 PMR/resources/etls/prime-pathologyDiagnoses.xml | 5 +----
 PMR/resources/etls/prime-weight.xml             | 2 +-
 8 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/PMR/resources/etls/prime-blooddraws.xml b/PMR/resources/etls/prime-blooddraws.xml
index 93d4a6cd..e9a49d6d 100644
--- a/PMR/resources/etls/prime-blooddraws.xml
+++ b/PMR/resources/etls/prime-blooddraws.xml
@@ -35,7 +35,7 @@
                     <setting name="destQuery" value="blood"/>
                     <setting name="destColumn" value="objectId"/>
 
-                    <setting name="reportOnly" value="true"/>
+                    <setting name="reportOnly" value="false"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-chemistryResults.xml b/PMR/resources/etls/prime-chemistryResults.xml
index bda1efe8..10d31856 100644
--- a/PMR/resources/etls/prime-chemistryResults.xml
+++ b/PMR/resources/etls/prime-chemistryResults.xml
@@ -40,7 +40,7 @@
                     <setting name="destQuery" value="chemistryResults"/>
                     <setting name="destColumn" value="objectId"/>
 
-                    <setting name="reportOnly" value="true"/>
+                    <setting name="reportOnly" value="false"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-clinpathRuns.xml b/PMR/resources/etls/prime-clinpathRuns.xml
index 70b9c6f0..d74ccc6e 100644
--- a/PMR/resources/etls/prime-clinpathRuns.xml
+++ b/PMR/resources/etls/prime-clinpathRuns.xml
@@ -42,7 +42,7 @@
                     <setting name="destQuery" value="clinpathRuns"/>
                     <setting name="destColumn" value="objectId"/>
 
-                    <setting name="reportOnly" value="true"/>
+                    <setting name="reportOnly" value="false"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-hematologyResults.xml b/PMR/resources/etls/prime-hematologyResults.xml
index e3aca8f3..4ba39f44 100644
--- a/PMR/resources/etls/prime-hematologyResults.xml
+++ b/PMR/resources/etls/prime-hematologyResults.xml
@@ -40,7 +40,7 @@
                     <setting name="destQuery" value="hematologyResults"/>
                     <setting name="destColumn" value="objectId"/>
 
-                    <setting name="reportOnly" value="true"/>
+                    <setting name="reportOnly" value="false"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-histology.xml b/PMR/resources/etls/prime-histology.xml
index b20efed0..1986eed7 100644
--- a/PMR/resources/etls/prime-histology.xml
+++ b/PMR/resources/etls/prime-histology.xml
@@ -40,7 +40,7 @@
                     <setting name="destQuery" value="histology"/>
                     <setting name="destColumn" value="objectId"/>
 
-                    <setting name="reportOnly" value="true"/>
+                    <setting name="reportOnly" value="false"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-microbiology.xml b/PMR/resources/etls/prime-microbiology.xml
index 0f7a74d7..73122f38 100644
--- a/PMR/resources/etls/prime-microbiology.xml
+++ b/PMR/resources/etls/prime-microbiology.xml
@@ -41,7 +41,7 @@
                     <setting name="destQuery" value="microbiology"/>
                     <setting name="destColumn" value="objectId"/>
 
-                    <setting name="reportOnly" value="true"/>
+                    <setting name="reportOnly" value="false"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-pathologyDiagnoses.xml b/PMR/resources/etls/prime-pathologyDiagnoses.xml
index aa9a38a6..efed3fb1 100644
--- a/PMR/resources/etls/prime-pathologyDiagnoses.xml
+++ b/PMR/resources/etls/prime-pathologyDiagnoses.xml
@@ -8,12 +8,9 @@
                 <sourceColumns>
                     <column>Id</column>
                     <column>date</column>
-                    <column>ageAtTime</column>
                     <column>sort_order</column>
                     <column>codes</column>
                     <column>objectid</column>
-                    <column>created</column>
-                    <column>modified</column>
                     <column>QCState/Label</column>
                 </sourceColumns>
                 <sourceFilters>
@@ -39,7 +36,7 @@
                     <setting name="destQuery" value="pathologyDiagnoses"/>
                     <setting name="destColumn" value="objectId"/>
 
-                    <setting name="reportOnly" value="true"/>
+                    <setting name="reportOnly" value="false"/>
                 </settings>
             </taskref>
         </transform>
diff --git a/PMR/resources/etls/prime-weight.xml b/PMR/resources/etls/prime-weight.xml
index e08c00c1..fd651cf4 100644
--- a/PMR/resources/etls/prime-weight.xml
+++ b/PMR/resources/etls/prime-weight.xml
@@ -38,7 +38,7 @@
                     <setting name="destQuery" value="weight"/>
                     <setting name="destColumn" value="objectId"/>
 
-                    <setting name="reportOnly" value="true"/>
+                    <setting name="reportOnly" value="false"/>
                 </settings>
             </taskref>
         </transform>

From 6de45c5d77e355700a6382953c70caed562258f9 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Tue, 7 Oct 2025 15:34:34 -0700
Subject: [PATCH 28/40] Update dependencies

---
 mcc/package-lock.json | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/mcc/package-lock.json b/mcc/package-lock.json
index b20dd7fb..a799d223 100644
--- a/mcc/package-lock.json
+++ b/mcc/package-lock.json
@@ -8193,10 +8193,11 @@
       }
     },
     "node_modules/tar-fs": {
-      "version": "2.1.3",
-      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.3.tgz",
-      "integrity": "sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg==",
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz",
+      "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "chownr": "^1.1.1",
         "mkdirp-classic": "^0.5.2",

From 8b980c1f849f300f9d63bb36150b5932d8f0e88f Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Wed, 15 Oct 2025 11:02:16 -0700
Subject: [PATCH 29/40] Update MCC text

---
 mcc/resources/views/mccRequests.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mcc/resources/views/mccRequests.html b/mcc/resources/views/mccRequests.html
index 53e77dce..f47a1ee7 100644
--- a/mcc/resources/views/mccRequests.html
+++ b/mcc/resources/views/mccRequests.html
@@ -8,7 +8,7 @@
 
         Ext4.get(webpart.wrapperDivId).update(
             'The MCC is now accepting animal requests! Investigators interested in requesting marmosets for their research can submit applications via the animal request portal.<br><br>' +
-            '<span style="font-weight: bold">Investigators can anticipate the estimated cost for getting marmosets to be about $5,500 USD per animal plus approximately $10,000 USD for shipping. Please note that this is an estimate and the actual cost is determined by the breeding centers and shipping can vary based on distance.</span>' +
+            '<span style="font-weight: bold">Investigators can anticipate the following estimated cost per animal: $5,500 USD for early-stage investigators, $6,500 USD for other academic investigators and $10,000 USD for commercial institutions. Arranging shipping is the responsibility of the requestor and is approximately $10,000 USD. Please note that this is an estimate and the actual cost is determined by the breeding centers and shipping can vary based on distance.</span>' +
             '<p></p>' +
             '<a class="labkey-text-link" href="<%=contextPath%>/mcc' + ctx['MCCRequestContainer'] + '/animalRequest.view">Submit New Animal Request</a><br>' +
             '<a class="labkey-text-link" href="/become-a-user.html#request-animals">Click Here to View Documentation on the Request Process and Scoring Criteria</a>' +

From 82c728eabf6f1355bbcd80439579a9d003575352 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Wed, 15 Oct 2025 14:02:42 -0700
Subject: [PATCH 30/40] Ensure extra memory for BWA-mem

---
 .../primeseq/pipeline/SequenceJobResourceAllocator.java      | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
index c213266f..cb15b069 100644
--- a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
+++ b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
@@ -96,6 +96,11 @@ private int getAlignerIndexMem(PipelineJob job)
                 }
             }
         }
+        else if (job.getClass().getName().endsWith("ReferenceLibraryPipelineJob"))
+        {
+            // This almost always includes bwa-mem
+            return 72;
+        }
 
         return 36;
     }

From d788191491866f93515b354cc07e94e2faec8566 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sat, 25 Oct 2025 08:11:02 -0700
Subject: [PATCH 31/40] Increase memory for kinship task

---
 .../primeseq/pipeline/SequenceJobResourceAllocator.java       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
index cb15b069..b7921c6a 100644
--- a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
+++ b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
@@ -184,8 +184,8 @@ public Integer getMaxRequestMemory(PipelineJob job)
 
         if (isGeneticsTask(job))
         {
-            job.getLogger().debug("setting memory to 72");
-            return 72;
+            job.getLogger().debug("setting memory to 96");
+            return 96;
         }
 
         if (isCacheAlignerIndexesTask(job))

From 084f20f9280370ffa1d9c517574c602638930587 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 27 Oct 2025 14:07:25 -0700
Subject: [PATCH 32/40] Misc small improvements to MCC

---
 mcc/resources/etls/mcc.xml                    |  1 +
 mcc/resources/etls/snprc-datasets.xml         |  2 +-
 .../MCC NIH Dashboard.folderType.xml          | 15 +++++
 .../queries/mcc/genomicDatasetsSource.sql     |  3 +-
 .../queries/study/genomicDatasets/.qview.xml  |  1 +
 .../study/datasets/datasets_metadata.xml      |  3 +
 mcc/resources/views/mccU24Demographics.html   |  5 +-
 mcc/src/client/Dashboard/Dashboard.tsx        |  4 +-
 mcc/src/client/GeneticsPlot/GeneticsPlot.tsx  | 61 +++++++++----------
 mcc/src/client/GeneticsPlot/KinshipTable.tsx  | 11 ++++
 mcc/src/client/GeneticsPlot/ScatterChart.tsx  | 11 ++++
 .../client/GeneticsPlot/SequenceDataTable.tsx | 39 ++++++++++++
 mcc/src/org/labkey/mcc/MccUserSchema.java     |  1 +
 .../mcc/etl/PopulateGeneticDataStep.java      |  3 +-
 14 files changed, 123 insertions(+), 37 deletions(-)
 create mode 100644 mcc/src/client/GeneticsPlot/SequenceDataTable.tsx

diff --git a/mcc/resources/etls/mcc.xml b/mcc/resources/etls/mcc.xml
index 9e3d71ad..d324ce40 100644
--- a/mcc/resources/etls/mcc.xml
+++ b/mcc/resources/etls/mcc.xml
@@ -86,6 +86,7 @@
                     <column>date</column>
                     <column>datatype</column>
                     <column>sra_accession</column>
+                    <column>total_reads</column>
                     <column>objectid</column>
                 </sourceColumns>
             </source>
diff --git a/mcc/resources/etls/snprc-datasets.xml b/mcc/resources/etls/snprc-datasets.xml
index 9b66a481..350a55d3 100644
--- a/mcc/resources/etls/snprc-datasets.xml
+++ b/mcc/resources/etls/snprc-datasets.xml
@@ -155,7 +155,7 @@
         <transform id="observations1" type="RemoteQueryTransformStep">
             <description>Copy to target</description>
             <source remoteSource="SNPRC" schemaName="study" queryName="u24MarmosetStats">
-                <!-- NOTE: the column orde must be preserved and match expected order in NprcObservationStep -->
+                <!-- NOTE: the column order must be preserved and match expected order in NprcObservationStep -->
                 <sourceColumns>
                     <column>AnimalId</column>
                     <column>Date</column>
diff --git a/mcc/resources/folderTypes/MCC NIH Dashboard.folderType.xml b/mcc/resources/folderTypes/MCC NIH Dashboard.folderType.xml
index 8bf4b486..9bea3b2e 100644
--- a/mcc/resources/folderTypes/MCC NIH Dashboard.folderType.xml	
+++ b/mcc/resources/folderTypes/MCC NIH Dashboard.folderType.xml	
@@ -74,6 +74,21 @@
                 </webPart>
             </preferredWebParts>
         </folderTab>
+        <folderTab>
+            <name>geneticsDashboard</name>
+            <caption>Genetics</caption>
+            <selectors>
+                <selector>
+                    <view>genetics</view>
+                </selector>
+            </selectors>
+            <preferredWebParts>
+                <webPart>
+                    <name>geneticsPlotWebpart</name>
+                    <location>body</location>
+                </webPart>
+            </preferredWebParts>
+        </folderTab>
         <folderTab>
             <name>requests</name>
             <caption>MCC Requests</caption>
diff --git a/mcc/resources/queries/mcc/genomicDatasetsSource.sql b/mcc/resources/queries/mcc/genomicDatasetsSource.sql
index 1dcdba77..332e76af 100644
--- a/mcc/resources/queries/mcc/genomicDatasetsSource.sql
+++ b/mcc/resources/queries/mcc/genomicDatasetsSource.sql
@@ -3,7 +3,8 @@ SELECT
     r.subjectid as Id,
     r.created as date,
     r.application as datatype,
-    r.sraRuns as sra_accession
+    r.sraRuns as sra_accession,
+    r.totalForwardReads as total_reads
 
 FROM sequenceanalysis.sequence_readsets r
 
diff --git a/mcc/resources/queries/study/genomicDatasets/.qview.xml b/mcc/resources/queries/study/genomicDatasets/.qview.xml
index e583f4f4..c29532b3 100644
--- a/mcc/resources/queries/study/genomicDatasets/.qview.xml
+++ b/mcc/resources/queries/study/genomicDatasets/.qview.xml
@@ -4,6 +4,7 @@
        <column name="date" />
        <column name="datatype" />
        <column name="sra_accession" />
+       <column name="total_reads" />
        <column name="history" />
    </columns>
    <sorts>
diff --git a/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml b/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml
index d1ce17b9..60b6cda9 100644
--- a/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml
+++ b/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml
@@ -1073,6 +1073,9 @@
             <column columnName="sra_accession">
                 <datatype>varchar</datatype>
             </column>
+            <column columnName="total_reads">
+                <datatype>integer</datatype>
+            </column>
             <column columnName="objectid">
                 <datatype>entityid</datatype>
                 <propertyURI>urn:ehr.labkey.org/#ObjectId</propertyURI>
diff --git a/mcc/resources/views/mccU24Demographics.html b/mcc/resources/views/mccU24Demographics.html
index e490ff8d..5afcd1cd 100644
--- a/mcc/resources/views/mccU24Demographics.html
+++ b/mcc/resources/views/mccU24Demographics.html
@@ -12,7 +12,10 @@
         title: 'MCC Animals',
         schemaName: 'study',
         queryName: 'demographics',
-        viewName: LABKEY.ActionURL.getParameter('viewName') ?? 'U24 Assigned'
+        viewName: LABKEY.ActionURL.getParameter('viewName'),
+        removeableFilters: [
+            LABKEY.Filter.create('u24_status', true, LABKEY.Filter.Types.EQUALS)
+        ]
     }).render(webpart.wrapperDivId);
 });
 
diff --git a/mcc/src/client/Dashboard/Dashboard.tsx b/mcc/src/client/Dashboard/Dashboard.tsx
index fe1ba520..382e95ad 100644
--- a/mcc/src/client/Dashboard/Dashboard.tsx
+++ b/mcc/src/client/Dashboard/Dashboard.tsx
@@ -88,9 +88,9 @@ export function Dashboard() {
                 </div>
                 <div className="col-md-4">
                     <div className="panel panel-default">
-                        <div className="panel-heading">Center (All Animals)</div>
+                        <div className="panel-heading">Center (Living Animals)</div>
                         <div className="panel-body">
-                            <PieChart fieldName = "colony" demographics={demographics} cutout = "30%" />
+                            <PieChart fieldName = "colony" demographics={living} cutout = "30%" />
                         </div>
                     </div>
                 </div>
diff --git a/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx b/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx
index 031af83d..b33734bb 100644
--- a/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx
+++ b/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx
@@ -5,21 +5,13 @@ import ScatterChart from './ScatterChart';
 import { Box, Tab, Tabs } from '@mui/material';
 import KinshipTable from './KinshipTable';
 import { ErrorBoundary } from '../components/ErrorBoundary';
+import SequenceDataTable from './SequenceDataTable';
 
 
-function GenomeBrowser(props: {jbrowseId: any}) {
-    const { jbrowseId } = props;
-
-    return (
-        <div>
-            <a href={ActionURL.buildURL('jbrowse', 'jbrowse', null, {session: jbrowseId})}>Click here to view Marmoset SNP data in the genome browser</a>
-        </div>
-    );
-}
-
 export function GeneticsPlot() {
     const [pcaData, setPcaData] = useState([]);
     const [kinshipData, setKinshipData] = useState([]);
+    const [sequenceData, setSequenceData] = useState([]);
     const [jbrowseId, setJBrowseId] = useState(null);
     const [value, setValue] = React.useState(0);
 
@@ -83,6 +75,29 @@ export function GeneticsPlot() {
             },
             scope: this
         });
+
+        Query.selectRows({
+            containerPath: containerPath,
+            schemaName: 'study',
+            queryName: 'genomicDatasets',
+            columns: 'Id,datatype,sra_accession,total_reads,objectid',
+            success: function(results) {
+                setSequenceData(results.rows.map((row) => {
+                    return({
+                        id: row.objectid,
+                        Id: row.Id,
+                        datatype: row.datatype,
+                        sra_accession: row.sra_accession,
+                        total_reads: row.total_reads
+                    })
+                }))
+            },
+            failure: function(response) {
+                alert('There was an error loading data');
+                console.log(response);
+            },
+            scope: this
+        });
     }, [] /* only run the effect on mount */);
 
     if (!containerPath) {
@@ -113,35 +128,19 @@ export function GeneticsPlot() {
                 578 marmosets on NCBI's Sequence Read Archive (SRA), we are excited to report that the MCC portal now
                 houses a call set with single nucleotide variants and short indels for over 800 individuals.
                 <p/>
-                The MCC genomic database is extensive, with each individual being genotype at millions of variants
-                across the genome. One way to summarize a large dataset can be done using Principal Component Analysis
-                (PCA). PCA is a technique used across disciplines (from astronomy to genomics) that reduces the
-                information in a multi-dimensional dataset to (fewer) principal components (PC) that retain overall
-                trends and patterns in the original data. Biologically, this could mean merging together two variants
-                that are always inherited together into just one PC, making the data easier to analyze while maintaining
-                its most important patterns. See the **Visualization with PCA** tab below.
-                <p/>
-                Although PCA is useful for broad-scale comparisons, it is not very useful when trying to distinguish
-                whether two individuals are siblings or first-cousins, for instance. For that, we have better statistics
-                that can describe the genetic relatedness between two individuals. We estimated genetic relatedness for
-                all pairs of individuals for which we have whole-genome data, and made these available under the
-                **Kinship** tab. There you will find the inferred relationships between pairs of individuals as well as
-                the calculated kinship coefficient, which is a quantitative measure of genetic relatedness
-                (see <a href="https://en.wikipedia.org/wiki/Coefficient_of_relationship#Kinship_coefficient">here</a> for more details).
-                <p/>
-                It is possible to explore the full MCC database of variants with a graphical interface by accessing the
-                **Genome Browser** tab. There you can, for example, visualize all the variants present in your gene of
-                interest by typing it's name in the search bar.
+                In addition to the information in the tabs below, you can use the MCC genome browser to view know variants and search by gene.
+                <a href={ActionURL.buildURL('jbrowse', 'jbrowse', null, {session: jbrowseId})}>Click here to view Marmoset SNP data in the genome browser</a>
                 <p/>
                 The genetic analyses described here were performed by Karina Ray (ONPRC), Murillo Rodrigues (ONPRC), and
                 Ric del Rosario (Broad Institute). Please contact us at <a href="mailto:mcc@ohsu.edu">mcc@ohsu.edu</a> with any
                 questions.
             </div>
+
             <Box sx={{ borderBottom: 1, borderColor: 'divider' }}>
                 <Tabs value={value} onChange={handleChange} aria-label="basic tabs example">
                     <Tab label="Population Genetic Diversity" {...a11yProps(0)} />
                     <Tab label="Kinship" {...a11yProps(1)} />
-                    <Tab label="Genetic Variants" {...a11yProps(2)} hidden={jbrowseId == null}/>
+                    <Tab label="Sequence Datasets" {...a11yProps(2)}/>
                 </Tabs>
             </Box>
             <div className="row">
@@ -150,7 +149,7 @@ export function GeneticsPlot() {
                         <div className="panel-body">
                             {value === 0 && <ScatterChart data={pcaData}/>}
                             {value === 1 && <KinshipTable data={kinshipData}/>}
-                            {value === 2 && <GenomeBrowser jbrowseId={jbrowseId}/>}
+                            {value === 2 && <SequenceDataTable data={sequenceData}/>}
                         </div>
                     </div>
                 </div>
diff --git a/mcc/src/client/GeneticsPlot/KinshipTable.tsx b/mcc/src/client/GeneticsPlot/KinshipTable.tsx
index ee4f316f..a95052ce 100644
--- a/mcc/src/client/GeneticsPlot/KinshipTable.tsx
+++ b/mcc/src/client/GeneticsPlot/KinshipTable.tsx
@@ -13,6 +13,16 @@ export default function KinshipTable(props: {data: any}) {
     ]
 
     return (
+        <>
+        <div style={{paddingBottom: 20, maxWidth: 1000}}>
+            Although PCA is useful for broad-scale comparisons, it is not very useful when trying to distinguish
+            whether two individuals are siblings or first-cousins, for instance. For that, we have better statistics
+            that can describe the genetic relatedness between two individuals. We estimated genetic relatedness for
+            all pairs of individuals for which we have whole-genome data, shown in the table below. There you will
+            find the inferred relationships between pairs of individuals as well as the calculated kinship coefficient,
+            which is a quantitative measure of genetic relatedness
+            (see <a href="https://en.wikipedia.org/wiki/Coefficient_of_relationship#Kinship_coefficient">here</a> for more details).
+        </div>
         <DataGrid
             autoHeight={true}
             columns={columns}
@@ -24,5 +34,6 @@ export default function KinshipTable(props: {data: any}) {
             paginationModel={pageModel}
             onPaginationModelChange={(model) => setPageModel(model)}
         />
+        </>
     );
 }
\ No newline at end of file
diff --git a/mcc/src/client/GeneticsPlot/ScatterChart.tsx b/mcc/src/client/GeneticsPlot/ScatterChart.tsx
index 3ffd23e3..1439e358 100644
--- a/mcc/src/client/GeneticsPlot/ScatterChart.tsx
+++ b/mcc/src/client/GeneticsPlot/ScatterChart.tsx
@@ -82,6 +82,17 @@ export default function ScatterChart(props: {data: any}) {
     }
 
     return (
+        <>
+        <div style={{paddingBottom: 20, maxWidth: 1000}}>
+            The MCC genomic database is extensive, with each individual being genotype at millions of variants
+            across the genome. One way to summarize a large dataset can be done using Principal Component Analysis
+            (PCA). PCA is a technique used across disciplines (from astronomy to genomics) that reduces the
+            information in a multi-dimensional dataset to (fewer) principal components (PC) that retain overall
+            trends and patterns in the original data. Biologically, this could mean merging together two variants
+            that are always inherited together into just one PC, making the data easier to analyze while maintaining
+            its most important patterns.
+        </div>
         <Scatter data={chartData} options={chartOptions}/>
+        </>
     );
 }
\ No newline at end of file
diff --git a/mcc/src/client/GeneticsPlot/SequenceDataTable.tsx b/mcc/src/client/GeneticsPlot/SequenceDataTable.tsx
new file mode 100644
index 00000000..6d137b4c
--- /dev/null
+++ b/mcc/src/client/GeneticsPlot/SequenceDataTable.tsx
@@ -0,0 +1,39 @@
+import React from 'react';
+import { DataGrid, GridColDef, GridPaginationModel, GridRenderCellParams, GridToolbar } from '@mui/x-data-grid';
+
+export default function SequenceDataTable(props: {data: any}) {
+    const { data } = props;
+    const [pageModel, setPageModel] = React.useState<GridPaginationModel>({page: 0, pageSize: 25});
+
+    const columns: GridColDef[] = [
+        { field: 'Id', headerName: 'Animal 1', width: 150, type: "string", headerAlign: 'left' },
+        { field: 'datatype', headerName: 'Datatype', width: 250, type: "string", headerAlign: 'left' },
+        { field: 'sra_accession', headerName: 'SRA Accession', width: 150, type: "string", headerAlign: 'right', renderCell: (params: GridRenderCellParams<any, string>) => {
+                return (
+                    <a
+                        target="_blank"
+                        href={params.value}
+                    >{params.value ? "https://trace.ncbi.nlm.nih.gov/Traces/sra/?run=" + params.value : ""}</a>
+                );
+            }},
+        { field: 'total_reads', headerName: 'Total Reads', width: 125, type: "number", headerAlign: 'left', flex: 1 }
+    ]
+
+    return (
+        <>
+        <div style={{paddingBottom: 20, maxWidth: 1000}}>
+        </div>
+        <DataGrid
+            autoHeight={true}
+            columns={columns}
+            rows={data}
+            slots={{
+                toolbar: GridToolbar
+            }}
+            pageSizeOptions={[10,25,50,100]}
+            paginationModel={pageModel}
+            onPaginationModelChange={(model) => setPageModel(model)}
+        />
+        </>
+    );
+}
\ No newline at end of file
diff --git a/mcc/src/org/labkey/mcc/MccUserSchema.java b/mcc/src/org/labkey/mcc/MccUserSchema.java
index 00481485..71912fc5 100644
--- a/mcc/src/org/labkey/mcc/MccUserSchema.java
+++ b/mcc/src/org/labkey/mcc/MccUserSchema.java
@@ -271,6 +271,7 @@ private TableInfo getGenomicsQuery()
                 "    d.date,\n" +
                 "    d.datatype,\n" +
                 "    d.sra_accession,\n" +
+                "    d.total_reads,\n" +
                 "    d.objectid,\n" +
                 "    d.container\n" +
                 "\n" +
diff --git a/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java b/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java
index 9e27a7c4..06feea72 100644
--- a/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java
+++ b/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java
@@ -68,7 +68,7 @@ private void populateGeneticData(PipelineJob job) throws PipelineJobException
         {
             //first select all rows from remote table
             SelectRowsCommand sr = new SelectRowsCommand(MccSchema.NAME, "genomicDatasetsSource");
-            sr.setColumns(Arrays.asList("Id", "date", "datatype", "sra_accession"));
+            sr.setColumns(Arrays.asList("Id", "date", "datatype", "sra_accession", "total_reads"));
 
             TableInfo aggregatedDemographics = QueryService.get().getUserSchema(job.getUser(), job.getContainer(), MccSchema.NAME).getTable("aggregatedDemographics");
 
@@ -104,6 +104,7 @@ private void populateGeneticData(PipelineJob job) throws PipelineJobException
                 newRow.put("date", x.get("date"));
                 newRow.put("datatype", x.get("datatype"));
                 newRow.put("sra_accession", x.get("sra_accession"));
+                newRow.put("total_reads", x.get("total_reads"));
 
                 toInsert.get(target).add(newRow);
             });

From 1f467b67a53fe56237ca0b659c09e2ec4a34ce2f Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 27 Oct 2025 14:40:11 -0700
Subject: [PATCH 33/40] bugfix to url

---
 mcc/src/client/GeneticsPlot/SequenceDataTable.tsx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mcc/src/client/GeneticsPlot/SequenceDataTable.tsx b/mcc/src/client/GeneticsPlot/SequenceDataTable.tsx
index 6d137b4c..6ec8b2f3 100644
--- a/mcc/src/client/GeneticsPlot/SequenceDataTable.tsx
+++ b/mcc/src/client/GeneticsPlot/SequenceDataTable.tsx
@@ -12,8 +12,8 @@ export default function SequenceDataTable(props: {data: any}) {
                 return (
                     <a
                         target="_blank"
-                        href={params.value}
-                    >{params.value ? "https://trace.ncbi.nlm.nih.gov/Traces/sra/?run=" + params.value : ""}</a>
+                        href={params.value ? "https://trace.ncbi.nlm.nih.gov/Traces/sra/?run=" + params.value : ""}
+                    >{params.value}</a>
                 );
             }},
         { field: 'total_reads', headerName: 'Total Reads', width: 125, type: "number", headerAlign: 'left', flex: 1 }

From 1fb01f72b0d85ebbaca8d7165644a480a54f98a4 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 27 Oct 2025 15:13:07 -0700
Subject: [PATCH 34/40] MCC webpart syntax fix

---
 mcc/resources/folderTypes/MCC NIH Dashboard.folderType.xml | 2 +-
 mcc/resources/queries/study/genomicDatasets.query.xml      | 4 ++++
 mcc/src/client/GeneticsPlot/webpart/app.tsx                | 2 +-
 mcc/src/client/GeneticsPlot/webpart/dev.tsx                | 2 +-
 4 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/mcc/resources/folderTypes/MCC NIH Dashboard.folderType.xml b/mcc/resources/folderTypes/MCC NIH Dashboard.folderType.xml
index 9bea3b2e..ae9d9e26 100644
--- a/mcc/resources/folderTypes/MCC NIH Dashboard.folderType.xml	
+++ b/mcc/resources/folderTypes/MCC NIH Dashboard.folderType.xml	
@@ -84,7 +84,7 @@
             </selectors>
             <preferredWebParts>
                 <webPart>
-                    <name>geneticsPlotWebpart</name>
+                    <name>Marmoset Genetics</name>
                     <location>body</location>
                 </webPart>
             </preferredWebParts>
diff --git a/mcc/resources/queries/study/genomicDatasets.query.xml b/mcc/resources/queries/study/genomicDatasets.query.xml
index c78b8546..fd9406c2 100644
--- a/mcc/resources/queries/study/genomicDatasets.query.xml
+++ b/mcc/resources/queries/study/genomicDatasets.query.xml
@@ -18,6 +18,10 @@
                         <columnTitle>SRA Accession</columnTitle>
                         <url>https://trace.ncbi.nlm.nih.gov/Traces/sra/?run=${sra_accession}</url>
                     </column>
+                    <column columnName="total_reads">
+                        <columnTitle>Total Reads</columnTitle>
+                        <formatString>#,##0.##</formatString>
+                    </column>
                 </columns>
             </table>
         </tables>
diff --git a/mcc/src/client/GeneticsPlot/webpart/app.tsx b/mcc/src/client/GeneticsPlot/webpart/app.tsx
index 0d923599..4b95f6c8 100644
--- a/mcc/src/client/GeneticsPlot/webpart/app.tsx
+++ b/mcc/src/client/GeneticsPlot/webpart/app.tsx
@@ -4,6 +4,6 @@ import { App } from '@labkey/api';
 
 import { GeneticsPlot } from '../GeneticsPlot';
 
-App.registerApp<any>('mccPcaWebpart', target => {
+App.registerApp<any>('geneticsPlotWebpart', target => {
     ReactDOM.render(<GeneticsPlot />, document.getElementById(target));
 });
diff --git a/mcc/src/client/GeneticsPlot/webpart/dev.tsx b/mcc/src/client/GeneticsPlot/webpart/dev.tsx
index a503bfdb..b4982017 100644
--- a/mcc/src/client/GeneticsPlot/webpart/dev.tsx
+++ b/mcc/src/client/GeneticsPlot/webpart/dev.tsx
@@ -4,6 +4,6 @@ import { App } from '@labkey/api';
 
 import { GeneticsPlot } from '../GeneticsPlot';
 
-App.registerApp<any>('mccPcaWebpart', target => {
+App.registerApp<any>('geneticsPlotWebpart', target => {
     ReactDOM.render(<GeneticsPlot />, document.getElementById(target));
 }, true);
\ No newline at end of file

From 7916bd8eff618867e1b1d9e89adee07bf51cac27 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 27 Oct 2025 15:44:48 -0700
Subject: [PATCH 35/40] MCC webpart syntax fix

---
 mcc/src/client/GeneticsPlot/GeneticsPlot.tsx | 20 ++++++++++++++------
 mcc/src/client/entryPoints.js                |  5 +++--
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx b/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx
index b33734bb..6a790821 100644
--- a/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx
+++ b/mcc/src/client/GeneticsPlot/GeneticsPlot.tsx
@@ -126,14 +126,22 @@ export function GeneticsPlot() {
                 Over the past few years, the MCC team has been working on extracting, sequencing and analyzing DNA from
                 marmosets across the participating breeding centers. While we have deposited the raw sequence data for
                 578 marmosets on NCBI's Sequence Read Archive (SRA), we are excited to report that the MCC portal now
-                houses a call set with single nucleotide variants and short indels for over 800 individuals.
-                <p/>
-                In addition to the information in the tabs below, you can use the MCC genome browser to view know variants and search by gene.
-                <a href={ActionURL.buildURL('jbrowse', 'jbrowse', null, {session: jbrowseId})}>Click here to view Marmoset SNP data in the genome browser</a>
-                <p/>
-                The genetic analyses described here were performed by Karina Ray (ONPRC), Murillo Rodrigues (ONPRC), and
+                houses a call set with single nucleotide variants and short indels for over 800 individuals. The genetic analyses
+                described here were performed by Karina Ray (ONPRC), Murillo Rodrigues (ONPRC), and
                 Ric del Rosario (Broad Institute). Please contact us at <a href="mailto:mcc@ohsu.edu">mcc@ohsu.edu</a> with any
                 questions.
+                <p/>
+                { jbrowseId ? (
+                    <>
+                        In addition to the information in the tabs below, you can use the MCC genome browser to view variants and/or search by gene:
+                        <p/>
+                        <ul>
+                            <li>
+                                <a style={{fontWeight: 'bold'}} href={ActionURL.buildURL('jbrowse', 'jbrowse', null, {session: jbrowseId})}>Click here to open the genome browser</a>
+                            </li>
+                        </ul>
+                    </>
+                ) : null }
             </div>
 
             <Box sx={{ borderBottom: 1, borderColor: 'divider' }}>
diff --git a/mcc/src/client/entryPoints.js b/mcc/src/client/entryPoints.js
index 9063de69..2f2cf21e 100644
--- a/mcc/src/client/entryPoints.js
+++ b/mcc/src/client/entryPoints.js
@@ -24,12 +24,13 @@ module.exports = {
     name: 'geneticsPlot',
     title: 'Marmoset Genetics',
     permissionClasses: ['org.labkey.api.security.permissions.ReadPermission'],
-    path: './src/client/GeneticsPlot'
+    path: './src/client/GeneticsPlot',
   }, {
     name: 'geneticsPlotWebpart',
     title: 'Marmoset Genetics',
     permissionClasses: ['org.labkey.api.security.permissions.ReadPermission'],
-    path: './src/client/GeneticsPlot/webpart'
+    path: './src/client/GeneticsPlot/webpart',
+    generateLib: true
   },{
     name: 'u24Dashboard',
     title: 'U24 Dashboard',

From c1f3405a3be119a5d1c2772819400ec244415074 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 27 Oct 2025 15:57:49 -0700
Subject: [PATCH 36/40] Add number formatting

---
 mcc/src/client/Dashboard/Dashboard.tsx    | 6 +++---
 mcc/src/client/U24Dashboard/Dashboard.tsx | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/mcc/src/client/Dashboard/Dashboard.tsx b/mcc/src/client/Dashboard/Dashboard.tsx
index 382e95ad..873da112 100644
--- a/mcc/src/client/Dashboard/Dashboard.tsx
+++ b/mcc/src/client/Dashboard/Dashboard.tsx
@@ -66,20 +66,20 @@ export function Dashboard() {
                         <div className="panel-heading">Census</div>
                         <div className="row">
                             <div className="panel-body count-panel-body">
-                                <div className="count-panel-text">{demographics.length}</div>
+                                <div className="count-panel-text">{new Intl.NumberFormat("en-IN").format(demographics.length)}</div>
                                 <div className="small text-muted">Marmosets tracked by MCC</div>
                             </div>
                         </div>
                         <div className="row mcc-col-centered">
                             <div className="col-md-3">
                                 <div className="panel-body count-panel-body">
-                                    <div className="count-panel-text-small">{living.length}</div>
+                                    <div className="count-panel-text-small">{new Intl.NumberFormat("en-IN").format(living.length)}</div>
                                     <div className="small text-muted text-center">Living</div>
                                 </div>
                             </div>
                             <div className="col-md-3">
                                 <div className="panel-body count-panel-body">
-                                    <div className="count-panel-text-small">{u24Assigned.length}</div>
+                                    <div className="count-panel-text-small">{new Intl.NumberFormat("en-IN").format(u24Assigned.length)}</div>
                                     <div className="small text-muted text-center">U24 Assigned</div>
                                 </div>
                             </div>
diff --git a/mcc/src/client/U24Dashboard/Dashboard.tsx b/mcc/src/client/U24Dashboard/Dashboard.tsx
index 8819a078..513ae0fa 100644
--- a/mcc/src/client/U24Dashboard/Dashboard.tsx
+++ b/mcc/src/client/U24Dashboard/Dashboard.tsx
@@ -147,20 +147,20 @@ export function Dashboard() {
                         <div className="panel-heading">U24 Census</div>
                         <div className="row">
                             <div className="panel-body count-panel-body">
-                                <div className="count-panel-text"><a href={ActionURL.buildURL("project", "begin", "", {pageId: "animalData"})}>{u24Assigned.length}</a></div>
+                                <div className="count-panel-text"><a href={ActionURL.buildURL("project", "begin", "", {pageId: "animalData"})}>{new Intl.NumberFormat("en-IN").format(u24Assigned.length)}</a></div>
                                 <div className="small text-muted text-center">Total U24 Animals</div>
                             </div>
                         </div>
                         <div className="row mcc-col-centered">
                             <div className="col-md-3">
                                 <div className="panel-body count-panel-body">
-                                    <div className="count-panel-text-small"><a href={ActionURL.buildURL("project", "begin", "", {pageId: "animalData", "u24.Availability~eq": "available for transfer"})}>{availableForTransfer.length}</a></div>
+                                    <div className="count-panel-text-small"><a href={ActionURL.buildURL("project", "begin", "", {pageId: "animalData", "u24.Availability~eq": "available for transfer"})}>{new Intl.NumberFormat("en-IN").format(availableForTransfer.length)}</a></div>
                                     <div className="small text-muted text-center">Available</div>
                                 </div>
                             </div>
                             <div className="col-md-3">
                                 <div className="panel-body count-panel-body">
-                                    <div className="count-panel-text-small"><a href={ActionURL.buildURL("project", "begin", "", {pageId: "requests"})}>{requestRows.length}</a></div>
+                                    <div className="count-panel-text-small"><a href={ActionURL.buildURL("project", "begin", "", {pageId: "requests"})}>{new Intl.NumberFormat("en-IN").format(requestRows.length)}</a></div>
                                     <div className="small text-muted text-center">Total Requests</div>
                                 </div>
                             </div>

From 95c14a3a998d4ef52edb5e06c5f1f0ba7b49bdec Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 27 Oct 2025 17:01:56 -0700
Subject: [PATCH 37/40] Collapse small colonies

---
 mcc/src/client/Dashboard/Dashboard.tsx        |  8 ++--
 mcc/src/client/U24Dashboard/Dashboard.tsx     | 16 +++----
 .../client/components/dashboard/PieChart.tsx  | 42 +++++++++++++------
 3 files changed, 41 insertions(+), 25 deletions(-)

diff --git a/mcc/src/client/Dashboard/Dashboard.tsx b/mcc/src/client/Dashboard/Dashboard.tsx
index 873da112..f9a2ba14 100644
--- a/mcc/src/client/Dashboard/Dashboard.tsx
+++ b/mcc/src/client/Dashboard/Dashboard.tsx
@@ -8,9 +8,9 @@ import PieChart from '../components/dashboard/PieChart';
 import BarChart from '../components/dashboard/BarChart';
 
 export function Dashboard() {
-    const [demographics, setDemographics] = useState(null);
-    const [living, setLiving] = useState(null);
-    const [u24Assigned, setu24Assigned] = useState(null);
+    const [demographics, setDemographics] = useState<[]>(null);
+    const [living, setLiving] = useState<[]>(null);
+    const [u24Assigned, setu24Assigned] = useState<[]>(null);
 
     const ctx = getServerContext().getModuleContext('mcc') || {};
     const containerPath = ctx.MCCContainer || null;
@@ -90,7 +90,7 @@ export function Dashboard() {
                     <div className="panel panel-default">
                         <div className="panel-heading">Center (Living Animals)</div>
                         <div className="panel-body">
-                            <PieChart fieldName = "colony" demographics={living} cutout = "30%" />
+                            <PieChart fieldName = "colony" demographics={living} cutout = "30%" collapseBelow = {0.025}  />
                         </div>
                     </div>
                 </div>
diff --git a/mcc/src/client/U24Dashboard/Dashboard.tsx b/mcc/src/client/U24Dashboard/Dashboard.tsx
index 513ae0fa..7ade780c 100644
--- a/mcc/src/client/U24Dashboard/Dashboard.tsx
+++ b/mcc/src/client/U24Dashboard/Dashboard.tsx
@@ -8,14 +8,14 @@ import BarChart, { ColorType } from '../components/dashboard/BarChart';
 import { ActiveElement, Chart, ChartEvent } from 'chart.js/dist/types/index';
 
 export function Dashboard() {
-    const [demographics, setDemographics] = useState(null);
-    const [living, setLiving] = useState(null);
-    const [u24Assigned, setu24Assigned] = useState(null);
-    const [availableForTransfer, setAvailableForTransfer] = useState(null);
-    const [requestRows, setRequestRows] = useState(null);
-    const [censusRows, setCensusRows] = useState(null);
-    const [birthData, setBirthData ] = useState(null);
-    const [breedingPairData, setBreedingPairData ] = useState(null);
+    const [demographics, setDemographics] = useState<[]>(null);
+    const [living, setLiving] = useState<[]>(null);
+    const [u24Assigned, setu24Assigned] = useState<[]>(null);
+    const [availableForTransfer, setAvailableForTransfer] = useState<[]>(null);
+    const [requestRows, setRequestRows] = useState<[]>(null);
+    const [censusRows, setCensusRows] = useState<[]>(null);
+    const [birthData, setBirthData ] = useState<[]>(null);
+    const [breedingPairData, setBreedingPairData ] = useState<[]>(null);
 
     const ctx = getServerContext().getModuleContext('mcc') || {};
     const containerPath = ctx.MCCContainer || null;
diff --git a/mcc/src/client/components/dashboard/PieChart.tsx b/mcc/src/client/components/dashboard/PieChart.tsx
index b9a28ec8..ca6880b4 100644
--- a/mcc/src/client/components/dashboard/PieChart.tsx
+++ b/mcc/src/client/components/dashboard/PieChart.tsx
@@ -1,11 +1,5 @@
 import React, { useEffect, useRef } from 'react';
-import {
-    Chart,
-    ArcElement,
-    Legend,
-    PieController,
-    Tooltip
-} from 'chart.js';
+import { ArcElement, Chart, Legend, PieController, Tooltip } from 'chart.js';
 
 Chart.register(ArcElement, Legend, PieController, Tooltip);
 
@@ -21,14 +15,12 @@ const colors = [
     "#999999"
 ];
 
-export default function PieChart(props) {
+export default function PieChart(props: {demographics: [], fieldName: string, cutout?: string, collapseBelow?: number }) {
     const canvas = useRef(null);
 
-    const { demographics } = props;
-    const { fieldName } = props;
-    const { cutout } = props || 0;
+    const { demographics, fieldName, cutout = '0', collapseBelow = 0 } = props;
 
-    const collectedData = demographics.reduce((acc, curr) => {
+    const collectedData  = demographics.reduce((acc, curr) => {
         const value = curr[fieldName] === null ? 'Unknown' : curr[fieldName];
         if (acc[value]) {
             acc[value] = acc[value] + 1;
@@ -37,7 +29,31 @@ export default function PieChart(props) {
         }
 
         return acc;
-    }, {});
+    }, new Map<string, bigint>())
+
+    if (collapseBelow) {
+        const total = Object.keys(collectedData).reduce((sum, keyName) => {
+            sum += collectedData[keyName]
+
+            return sum
+        }, 0)
+
+        const otherValue = Object.keys(collectedData).reduce((sum, keyName) => {
+            const val = collectedData[keyName]
+            const fraction = val / total
+            if (fraction < collapseBelow) {
+                delete collectedData[keyName]
+                sum += val
+            }
+
+            return sum
+        }, 0)
+
+        if (otherValue) {
+            collectedData['Other'] = otherValue
+        }
+    }
+
     const labels = Object.keys(collectedData).sort(Intl.Collator().compare);
     const data = labels.map(label => collectedData[label]);
 

From e6b35986f0afc00f2b0becc4fdcbe35b7c437b0d Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sun, 2 Nov 2025 05:51:29 -0800
Subject: [PATCH 38/40] Improve behavior of MCC MarkShippedWindow

---
 .../web/mcc/window/MarkShippedWindow.js       | 343 +++++++++++-------
 1 file changed, 214 insertions(+), 129 deletions(-)

diff --git a/mcc/resources/web/mcc/window/MarkShippedWindow.js b/mcc/resources/web/mcc/window/MarkShippedWindow.js
index afa44eaa..a38fc3ab 100644
--- a/mcc/resources/web/mcc/window/MarkShippedWindow.js
+++ b/mcc/resources/web/mcc/window/MarkShippedWindow.js
@@ -208,7 +208,6 @@ Ext4.define('MCC.window.MarkShippedWindow', {
             return;
         }
 
-        var targetFolderId = win.down('#targetFolder').store.findRecord('Path', targetFolder).get('EntityId');
         Ext4.Msg.wait('Saving...');
         LABKEY.Query.selectRows({
             schemaName: 'study',
@@ -224,143 +223,229 @@ Ext4.define('MCC.window.MarkShippedWindow', {
                     return false;
                 }
 
-                var commands = [];
-                Ext4.Array.forEach(results.rows, function(row){
-                    var effectiveId = win.down('#usePreviousId-' + row.Id).getValue() ? row.Id : win.down('#newId-' + row.Id).getValue();
-                    var requestId = win.down('#requestId-' + row.Id).getValue();
-                    // This should be checked above, although perhaps case sensitivity could get involved:
-                    LDK.Assert.assertNotEmpty('Missing effective ID after query', effectiveId);
-
-                    var shouldAddDeparture = !row['Id/MostRecentDeparture/MostRecentDeparture'] ||
-                            row['Id/MostRecentDeparture/MostRecentDeparture'] !== Ext4.Date.format(row.effectiveDate, 'Y-m-d') ||
-                            row['Id/MostRecentDeparture/mccRequestId'] !== requestId ||
-                            row.Id !== effectiveId;
-                    if (shouldAddDeparture) {
-                        commands.push({
-                            command: 'insert',
-                            schemaName: 'study',
-                            queryName: 'Departure',
-                            rows: [{
-                                Id: row.Id,
-                                date: effectiveDate,
-                                source: row.colony,
-                                destination: centerName,
-                                mccRequestId: requestId,
-                                description: row.colony ? 'Original center: ' + row.colony : null,
-                                qcstate: null,
-                                objectId: null,
-                                QCStateLabel: 'Completed'
-                            }]
-                        });
-                    }
+                var uniqueIds = [];
+                Ext4.Array.forEach(results.rows, function(row) {
+                    uniqueIds.push(win.down('#usePreviousId-' + row.Id).getValue() ? row.Id : win.down('#newId-' + row.Id).getValue());
+                }, this);
 
-                    // If going to a new LK folder, we're creating a whole new record:
-                    if (targetFolderId.toUpperCase() !== LABKEY.Security.currentContainer.id.toUpperCase() || effectiveId !== row.Id) {
-                        commands.push({
-                            command: 'insert',
-                            containerPath: targetFolder,
-                            schemaName: 'study',
-                            queryName: 'Demographics',
-                            rows: [{
-                                Id: effectiveId,
-                                date: effectiveDate,
-                                alternateIds: row.Id !== effectiveId ? row.Id : null,
-                                gender: row.gender,
-                                species: row.species,
-                                birth: row.birth,
-                                death: row.death,
-                                dam: row.dam,
-                                sire: row.sire,
-                                damMccAlias: row['damMccAlias/externalAlias'],
-                                sireMccAlias: row['sireMccAlias/externalAlias'],
-                                colony: centerName,
-                                source: row.colony,
-                                calculated_status: 'Alive',
-                                mccAlias: row['Id/mccAlias/externalAlias'],
-                                QCState: null,
-                                QCStateLabel: 'Completed',
-                                objectId: null
-                            }]
-                        });
-
-                        commands.push({
-                            command: 'update',
-                            containerPath: null, //Use current folder
-                            schemaName: 'study',
-                            queryName: 'Demographics',
-                            rows: [{
-                                Id: row.Id, // NOTE: always change the original record
-                                excludeFromCensus: true
-                            }]
-                        });
-                    }
-                    else {
-                        // Otherwise update the existing:
-                        commands.push({
-                            command: 'update',
-                            containerPath: targetFolder,
-                            schemaName: 'study',
-                            queryName: 'Demographics',
-                            rows: [{
-                                Id: row.Id,
-                                date: effectiveDate,
-                                alternateIds: null,
-                                gender: row.gender,
-                                species: row.species,
-                                birth: row.birth,
-                                death: row.death,
-                                dam: row.dam,
-                                sire: row.sire,
-                                colony: centerName,
-                                source: row.colony,
-                                calculated_status: 'Alive',
-                                QCState: null,
-                                QCStateLabel: 'Completed',
-                                objectId: null
-                            }]
-                        });
+                LABKEY.Query.SelectRows({
+                    schemaName: 'study',
+                    queryName: 'Demographics',
+                    containerPath: targetFolder,
+                    filterArray: [LABKEY.Filter.create('Id', uniqueIds.join(';'), LABKEY.Filter.Types.IN)],
+                    columns: 'Id,gender,species,birth,death,dam,sire,damMccAlias/externalAlias,sireMccAlias/externalAlias,calculated_status,Id/mccAlias/externalAlias,colony,source,lsid,objectid',
+                    scope: this,
+                    failure: LDK.Utils.getErrorCallback(),
+                    success: function(existingIdResults) {
+                        var preexistingIdsInTargetFolder = {};
+                        Ext4.Array.forEach(existingIdResults.rows, function(r){
+                            preexistingIdsInTargetFolder[r.Id] = r;
+                        }, this);
+
+                        this.doSave(win, results, preexistingIdsInTargetFolder);
                     }
+                });
+            }
+        });
+    },
+
+    doSave: function(win, results, preexistingIdsInTargetFolder){
+        var effectiveDate = win.down('#effectiveDate').getValue();
+        var centerName = win.down('#centerName').getValue();
+        var targetFolder = win.down('#targetFolder').getValue();
+        var targetFolderId = win.down('#targetFolder').store.findRecord('Path', targetFolder).get('EntityId');
+
+        var commands = [];
+        var hadError = false;
+        Ext4.Array.forEach(results.rows, function(row){
+            var effectiveId = win.down('#usePreviousId-' + row.Id).getValue() ? row.Id : win.down('#newId-' + row.Id).getValue();
+            var requestId = win.down('#requestId-' + row.Id).getValue();
+            // This should be checked above, although perhaps case sensitivity could get involved:
+            LDK.Assert.assertNotEmpty('Missing effective ID after query', effectiveId);
+
+            var shouldAddDeparture = !row['Id/MostRecentDeparture/MostRecentDeparture'] ||
+                    row['Id/MostRecentDeparture/MostRecentDeparture'] !== Ext4.Date.format(row.effectiveDate, 'Y-m-d') ||
+                    row['Id/MostRecentDeparture/mccRequestId'] !== requestId ||
+                    row.Id !== effectiveId;
+            if (shouldAddDeparture) {
+                commands.push({
+                    command: 'insert',
+                    schemaName: 'study',
+                    queryName: 'Departure',
+                    rows: [{
+                        Id: row.Id,
+                        date: effectiveDate,
+                        source: row.colony,
+                        destination: centerName,
+                        mccRequestId: requestId,
+                        description: row.colony ? 'Original center: ' + row.colony : null,
+                        qcstate: null,
+                        objectId: null,
+                        QCStateLabel: 'Completed'
+                    }]
+                });
+            }
 
-                    var shouldAddArrival = !row['Id/MostRecentArrival/MostRecentArrival'] ||
-                            row['Id/MostRecentArrival/MostRecentArrival'] !== Ext4.Date.format(row.effectiveDate, 'Y-m-d') ||
-                            row['Id/MostRecentArrival/mccRequestId'] !== requestId ||
-                            row.Id !== effectiveId;
-                    if (shouldAddArrival) {
-                        // And also add an arrival record. NOTE: set the date after the departure to get status to update properly
-                        var arrivalDate = new Date(effectiveDate).setMinutes(effectiveDate.getMinutes() + 1);
-                        commands.push({
-                            command: 'insert',
-                            containerPath: targetFolder,
-                            schemaName: 'study',
-                            queryName: 'Arrival',
-                            rows: [{
-                                Id: effectiveId,
-                                date: arrivalDate,
-                                source: centerName,
-                                mccRequestId: requestId,
-                                QCState: null,
-                                QCStateLabel: 'Completed',
-                                objectId: null
-                            }]
-                        });
+            // If going to a new LK folder, we're creating a whole new record:
+            if (targetFolderId.toUpperCase() !== LABKEY.Security.currentContainer.id.toUpperCase() || effectiveId !== row.Id) {
+                if (Ext4.Object.getKeys(preexistingIdsInTargetFolder).indexOf(effectiveId) === -1) {
+                    // No existing record for this ID, make new record:
+                    commands.push({
+                        command: 'insert',
+                        containerPath: targetFolder,
+                        schemaName: 'study',
+                        queryName: 'Demographics',
+                        rows: [{
+                            Id: effectiveId,
+                            date: effectiveDate,
+                            alternateIds: row.Id !== effectiveId ? row.Id : null,
+                            gender: row.gender,
+                            species: row.species,
+                            birth: row.birth,
+                            death: row.death,
+                            dam: row.dam,
+                            sire: row.sire,
+                            damMccAlias: row['damMccAlias/externalAlias'],
+                            sireMccAlias: row['sireMccAlias/externalAlias'],
+                            colony: centerName,
+                            source: row.colony,
+                            calculated_status: 'Alive',
+                            mccAlias: row['Id/mccAlias/externalAlias'],
+                            QCState: null,
+                            QCStateLabel: 'Completed',
+                            objectId: null
+                        }]
+                    });
+                }
+                else {
+                    // There is an existing record for this ID, so merge/validate:
+                    console.log('Existing record found for: ' + effectiveId)
+                    var toUpdate = preexistingIdsInTargetFolder[effectiveId]
+
+                    var errors = []
+                    Ext4.Array.forEach(['gender', 'species', 'birth', 'death', 'dam', 'sire'], function(fieldName) {
+                        this.doFieldCheck(row, fieldName, toUpdate, fieldName, errors, effectiveId)
+                    }, this);
+
+                    toUpdate.colony = centerName
+                    toUpdate.source = toUpdate.source || row.colony
+                    toUpdate.calculated_status = toUpdate.calculated_status || 'Alive';
+
+                    if (row.Id !== effectiveId) {
+                        toUpdate.alternateIds = toUpdate.alternateIds ? toUpdate.alternateIds + ',' + row.Id : row.Id;
                     }
-                }, this);
 
-                LABKEY.Query.saveRows({
-                    commands: commands,
-                    scope: this,
-                    failure: LDK.Utils.getErrorCallback(),
-                    success: function() {
-                        Ext4.Msg.hide();
-                        Ext4.Msg.alert('Success', 'Transfer Added', function () {
-                            var dataRegion = LABKEY.DataRegions[this.dataRegionName];
-                            this.destroy();
+                    this.doFieldCheck(row, 'damMccAlias/externalAlias', toUpdate, 'damMccAlias', errors, effectiveId)
+                    this.doFieldCheck(row, 'sireMccAlias/externalAlias', toUpdate, 'sireMccAlias', errors, effectiveId)
+                    this.doFieldCheck(row, 'Id/mccAlias/externalAlias', toUpdate, 'mccAlias', errors, effectiveId)
 
-                            dataRegion.refresh();
-                        }, this);
+                    if (errors.length) {
+                        Ext4.Msg.hide();
+                        Ext4.Msg.alert('Error', 'Inconsistent data between source and destination demographics for: ' + effectiveId + + '<br>' + errors.join('<br>'));
+                        hadError = true;
+                        return false;
                     }
+
+                    commands.push({
+                        command: 'update',
+                        containerPath: targetFolder,
+                        schemaName: 'study',
+                        queryName: 'Demographics',
+                        rows: [toUpdate]
+                    });
+                }
+
+                commands.push({
+                    command: 'update',
+                    containerPath: null, //Use current folder
+                    schemaName: 'study',
+                    queryName: 'Demographics',
+                    rows: [{
+                        Id: row.Id, // NOTE: always change the original record
+                        excludeFromCensus: true
+                    }]
                 });
             }
+            else {
+                // Otherwise update the existing:
+                commands.push({
+                    command: 'update',
+                    containerPath: targetFolder,
+                    schemaName: 'study',
+                    queryName: 'Demographics',
+                    rows: [{
+                        Id: row.Id,
+                        date: effectiveDate,
+                        alternateIds: null,
+                        gender: row.gender,
+                        species: row.species,
+                        birth: row.birth,
+                        death: row.death,
+                        dam: row.dam,
+                        sire: row.sire,
+                        colony: centerName,
+                        source: row.colony,
+                        calculated_status: 'Alive',
+                        QCState: null,
+                        QCStateLabel: 'Completed',
+                        objectId: null
+                    }]
+                });
+            }
+
+            var shouldAddArrival = !row['Id/MostRecentArrival/MostRecentArrival'] ||
+                    row['Id/MostRecentArrival/MostRecentArrival'] !== Ext4.Date.format(row.effectiveDate, 'Y-m-d') ||
+                    row['Id/MostRecentArrival/mccRequestId'] !== requestId ||
+                    row.Id !== effectiveId;
+            if (shouldAddArrival) {
+                // And also add an arrival record. NOTE: set the date after the departure to get status to update properly
+                var arrivalDate = new Date(effectiveDate).setMinutes(effectiveDate.getMinutes() + 1);
+                commands.push({
+                    command: 'insert',
+                    containerPath: targetFolder,
+                    schemaName: 'study',
+                    queryName: 'Arrival',
+                    rows: [{
+                        Id: effectiveId,
+                        date: arrivalDate,
+                        source: centerName,
+                        mccRequestId: requestId,
+                        QCState: null,
+                        QCStateLabel: 'Completed',
+                        objectId: null
+                    }]
+                });
+            }
+        }, this);
+
+        if (hadError) {
+            return;
+        }
+
+        LABKEY.Query.saveRows({
+            commands: commands,
+            scope: this,
+            failure: LDK.Utils.getErrorCallback(),
+            success: function() {
+                Ext4.Msg.hide();
+                Ext4.Msg.alert('Success', 'Transfer Added', function () {
+                    var dataRegion = LABKEY.DataRegions[this.dataRegionName];
+                    this.destroy();
+
+                    dataRegion.refresh();
+                }, this);
+            }
         });
+    },
+
+    doFieldCheck: function(row, fieldName1, toUpdate, fieldName2, errors, effectiveId) {
+        if (row[fieldName1]) {
+            if (toUpdate[fieldName2] && toUpdate[fieldName2] !== row[fieldName1]) {
+                errors.push('Pre-existing record for ' + effectiveId + ', but ' + fieldName2 + ' was inconsistent between old/new (' + toUpdate[fieldName2] + '/' + row[fieldName1] + ')')
+            }
+            else {
+                toUpdate[fieldName2] = row[fieldName1];
+            }
+        }
     }
 });
\ No newline at end of file

From 0496cb387ea1475db13f731a0db5314a2d8d7173 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sun, 2 Nov 2025 06:30:35 -0800
Subject: [PATCH 39/40] Bugfix to MarkShippedWindow

---
 mcc/resources/web/mcc/window/MarkShippedWindow.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mcc/resources/web/mcc/window/MarkShippedWindow.js b/mcc/resources/web/mcc/window/MarkShippedWindow.js
index a38fc3ab..9cbf6818 100644
--- a/mcc/resources/web/mcc/window/MarkShippedWindow.js
+++ b/mcc/resources/web/mcc/window/MarkShippedWindow.js
@@ -228,7 +228,7 @@ Ext4.define('MCC.window.MarkShippedWindow', {
                     uniqueIds.push(win.down('#usePreviousId-' + row.Id).getValue() ? row.Id : win.down('#newId-' + row.Id).getValue());
                 }, this);
 
-                LABKEY.Query.SelectRows({
+                LABKEY.Query.selectRows({
                     schemaName: 'study',
                     queryName: 'Demographics',
                     containerPath: targetFolder,

From 16681ba801bdba513bfdc8c504aaeb3cc473eba2 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Wed, 5 Nov 2025 07:14:26 -0800
Subject: [PATCH 40/40] Auto-create demographics records from SIV study
 assignment

---
 .../SivStudiesDataValidationNotification.java |  35 ++++-
 .../query/AutoCreateDemographicsTrigger.java  | 127 ++++++++++++++++++
 .../query/DefaultDatasetTrigger.java          |  29 ++++
 .../query/SivStudiesCustomizer.java           |   1 +
 mcc/resources/etls/mcc.xml                    |   2 +-
 5 files changed, 192 insertions(+), 2 deletions(-)
 create mode 100644 SivStudies/src/org/labkey/sivstudies/query/AutoCreateDemographicsTrigger.java

diff --git a/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java b/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java
index faeae93c..24c03dbb 100644
--- a/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java
+++ b/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java
@@ -1,20 +1,30 @@
 package org.labkey.sivstudies.notification;
 
+import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.Nullable;
+import org.labkey.api.data.CompareType;
 import org.labkey.api.data.Container;
+import org.labkey.api.data.SimpleFilter;
 import org.labkey.api.data.TableInfo;
 import org.labkey.api.data.TableSelector;
 import org.labkey.api.ldk.notification.AbstractNotification;
 import org.labkey.api.module.ModuleLoader;
+import org.labkey.api.query.FieldKey;
 import org.labkey.api.query.QueryService;
 import org.labkey.api.query.UserSchema;
 import org.labkey.api.security.User;
+import org.labkey.api.study.Study;
+import org.labkey.api.study.StudyService;
+import org.labkey.api.util.logging.LogHelper;
 import org.labkey.sivstudies.SivStudiesModule;
+import org.labkey.sivstudies.query.DefaultDatasetTrigger;
 
 import java.util.Date;
 
 public class SivStudiesDataValidationNotification extends AbstractNotification
 {
+    protected static final Logger _log = LogHelper.getLogger(DefaultDatasetTrigger.class, "Messages related to SivStudiesDataValidationNotification");
+
     public SivStudiesDataValidationNotification()
     {
         super(ModuleLoader.getInstance().getModule(SivStudiesModule.class));
@@ -65,6 +75,7 @@ public String getEmailSubject(Container c)
         duplicateInfectionCheck(c, u, msg);
         infectionAnchorDateDiscordance(c, u, msg);
         pvlWithoutInfectionDate(c, u, msg);
+        idsMissingFromDemographics(c, u, msg);
 
         if (!msg.isEmpty())
         {
@@ -102,10 +113,15 @@ private void infectionAnchorDateDiscordance(Container c, User u, StringBuilder m
     }
 
     private void genericQueryCheck(Container c, User u, StringBuilder msg, String schemaName, String queryName, String message)
+    {
+        genericQueryCheck(c, u, msg, schemaName, queryName, message, null);
+    }
+
+    private void genericQueryCheck(Container c, User u, StringBuilder msg, String schemaName, String queryName, String message, @Nullable SimpleFilter filter)
     {
         TableInfo ti = getTableInfo(u, c, schemaName, queryName);
 
-        TableSelector ts = new TableSelector(ti);
+        TableSelector ts = new TableSelector(ti, filter, null);
         long count = ts.getRowCount();
         if (count > 0)
         {
@@ -119,4 +135,21 @@ private void pvlWithoutInfectionDate(Container c, User u, StringBuilder msg)
     {
         genericQueryCheck(c, u, msg, "study", "pvlWithoutInfectionDate", "animals with PVL data but no record of SIV infection");
     }
+
+    private void idsMissingFromDemographics(Container c, User u, StringBuilder msg)
+    {
+        Study s = StudyService.get().getStudy(getTargetContainer(c));
+        if (s == null)
+        {
+            return;
+        }
+
+        SimpleFilter filter = new SimpleFilter(FieldKey.fromString("DataSet/Demographics/" + s.getSubjectColumnName()), null, CompareType.ISBLANK);
+        genericQueryCheck(c, u, msg, "study", s.getSubjectNounSingular(), "IDs with data in the study not present in the demographics table", filter);
+    }
+
+    protected Container getTargetContainer(Container c)
+    {
+        return c.isWorkbookOrTab() ? c.getParent() : c;
+    }
 }
diff --git a/SivStudies/src/org/labkey/sivstudies/query/AutoCreateDemographicsTrigger.java b/SivStudies/src/org/labkey/sivstudies/query/AutoCreateDemographicsTrigger.java
new file mode 100644
index 00000000..00e5b739
--- /dev/null
+++ b/SivStudies/src/org/labkey/sivstudies/query/AutoCreateDemographicsTrigger.java
@@ -0,0 +1,127 @@
+package org.labkey.sivstudies.query;
+
+import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.labkey.api.collections.CaseInsensitiveHashSet;
+import org.labkey.api.data.CompareType;
+import org.labkey.api.data.Container;
+import org.labkey.api.data.SimpleFilter;
+import org.labkey.api.data.TableInfo;
+import org.labkey.api.data.TableSelector;
+import org.labkey.api.data.triggers.Trigger;
+import org.labkey.api.data.triggers.TriggerFactory;
+import org.labkey.api.query.BatchValidationException;
+import org.labkey.api.query.DuplicateKeyException;
+import org.labkey.api.query.FieldKey;
+import org.labkey.api.query.QueryService;
+import org.labkey.api.query.QueryUpdateServiceException;
+import org.labkey.api.query.ValidationException;
+import org.labkey.api.security.User;
+import org.labkey.api.study.Study;
+import org.labkey.api.study.StudyService;
+import org.labkey.api.util.PageFlowUtil;
+import org.labkey.api.util.logging.LogHelper;
+
+import java.sql.SQLException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+public class AutoCreateDemographicsTrigger extends DefaultDatasetTrigger
+{
+    protected static final Logger _log = LogHelper.getLogger(DefaultDatasetTrigger.class, "Messages related to CreateDemographicsTrigger");
+
+    public static class Factory implements TriggerFactory
+    {
+        public Factory()
+        {
+
+        }
+
+        @Override
+        public @NotNull Collection<Trigger> createTrigger(@Nullable Container c, TableInfo table, Map<String, Object> extraContext)
+        {
+            return List.of(new AutoCreateDemographicsTrigger());
+        }
+    }
+
+    private static final String CACHE_KEY = "~~AutoCreateDemographicsTrigger.IdsToCreate~~";
+
+    @Override
+    protected void afterUpsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, @Nullable Map<String, Object> oldRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
+    {
+        if (extraContext == null)
+        {
+            _log.error("extraContext is null in AutoCreateDemographicsTrigger.afterUpsert()");
+            return;
+        }
+
+        if (!extraContext.containsKey(AutoCreateDemographicsTrigger.CACHE_KEY))
+        {
+            extraContext.put(CACHE_KEY, new CaseInsensitiveHashSet());
+        }
+
+        if (extraContext.get(CACHE_KEY) instanceof CaseInsensitiveHashSet s)
+        {
+            String idField = getIdField(c);
+            String id = newRow.get(idField) != null  ? newRow.get(idField).toString() : null;
+            if (id != null)
+            {
+                s.add(id);
+            }
+        }
+    }
+
+    @Override
+    public void complete(TableInfo table, Container c, User user, TableInfo.TriggerType event, BatchValidationException errors, Map<String, Object> extraContext)
+    {
+        if (extraContext == null)
+        {
+            _log.error("extraContext is null in AutoCreateDemographicsTrigger.complete()");
+            return;
+        }
+
+        if (extraContext.get(CACHE_KEY) instanceof CaseInsensitiveHashSet s)
+        {
+            s = new CaseInsensitiveHashSet(s);
+
+            String idField = getIdField(c);
+            TableInfo ti = QueryService.get().getUserSchema(user, getTargetContainer(c), "study").getTable("demographics");
+            List<String> existingIds = new TableSelector(ti, PageFlowUtil.set(idField), new SimpleFilter(FieldKey.fromString(idField), s, CompareType.IN), null).getArrayList(String.class);
+
+            s.removeAll(existingIds);
+
+            if (!s.isEmpty())
+            {
+                List<Map<String, Object>> toInsert = s.stream().map(id -> Map.of(idField, (Object)id)).toList();
+                try
+                {
+                    ti.getUpdateService().insertRows(user, c, toInsert, null, null, null);
+                }
+                catch (SQLException | BatchValidationException | QueryUpdateServiceException | DuplicateKeyException e)
+                {
+                    _log.error("Error creating demographics records", e);
+                }
+            }
+        }
+    }
+
+    private String _idField = null;
+
+    private String getIdField(Container c)
+    {
+        if (_idField == null)
+        {
+            Study s = StudyService.get().getStudy(getTargetContainer(c));
+            if (s == null)
+            {
+                return null;
+            }
+
+            _idField = s.getSubjectColumnName();
+        }
+
+        return _idField;
+    }
+}
diff --git a/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java b/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java
index 26efb191..fea3c889 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java
@@ -40,6 +40,29 @@ public void beforeInsert(TableInfo table, Container c, User user, @Nullable Map<
         beforeInsert(table, c, user, newRow, errors, extraContext, null);
     }
 
+    @Override
+    public void afterInsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, ValidationException errors, Map<String, Object> extraContext, @Nullable Map<String, Object> existingRecord) throws ValidationException
+    {
+        afterUpsert(table, c, user, newRow, existingRecord, errors, extraContext);
+    }
+
+    @Override
+    public void afterInsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
+    {
+        afterInsert(table, c, user, newRow, errors, extraContext, null);
+    }
+
+    @Override
+    public void afterUpdate(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, @Nullable Map<String, Object> oldRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
+    {
+        afterUpsert(table, c, user, newRow, oldRow, errors, extraContext);
+    }
+
+    protected void afterUpsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, @Nullable Map<String, Object> oldRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
+    {
+
+    }
+
     @Override
     public void beforeInsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, ValidationException errors, Map<String, Object> extraContext, @Nullable Map<String, Object> existingRecord) throws ValidationException
     {
@@ -84,4 +107,10 @@ private void mergeOldToNewRow(@NotNull Map<String, Object> newRow, @Nullable Map
             }
         }
     }
+
+    protected Container getTargetContainer(Container c)
+    {
+        return c.isWorkbookOrTab() ? c.getParent() : c;
+    }
+
 }
diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
index 4a8f3e36..1fc15ddb 100644
--- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
+++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java
@@ -79,6 +79,7 @@ public void performDatasetCustomization(DatasetTable ds)
             if ("assignment".equalsIgnoreCase(ds.getName()))
             {
                 ati.addTriggerFactory(StudiesService.get().getStudiesTriggerFactory());
+                ati.addTriggerFactory(new AutoCreateDemographicsTrigger.Factory());
             }
         }
         else
diff --git a/mcc/resources/etls/mcc.xml b/mcc/resources/etls/mcc.xml
index d324ce40..e5a78873 100644
--- a/mcc/resources/etls/mcc.xml
+++ b/mcc/resources/etls/mcc.xml
@@ -38,7 +38,7 @@
                     <column>objectid</column>
                 </sourceColumns>
             </source>
-            <destination schemaName="study" queryName="Demographics" targetOption="truncate" bulkLoad="true" batchSize="5000">
+            <destination schemaName="study" queryName="Demographics" targetOption="truncate" bulkLoad="true" batchSize="2500">
                 <alternateKeys>
                     <column name="objectid"/>
                 </alternateKeys>