From 5d732b6ad3945ecbb3106321c6c4f7b5dc61f862 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 4 Sep 2025 15:52:07 +0200 Subject: [PATCH 01/25] GPU TPC: Remove option to retry refit if cluster rejection breaks the track, we will anyway rebuild the track in the future --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 - .../Global/GPUChainTrackingMerger.cxx | 3 -- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 3 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 3 -- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 7 ++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 49 +++++-------------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 4 +- 7 files changed, 17 insertions(+), 53 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index bbf8dbb508b4a..8b08fcc648143 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -155,7 +155,6 @@ AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation inst AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF") AddOptionRTC(mergerNonInterpolateRejectMinNDF, uint8_t, 5, "", 0, "Minimum NDF of track for non-interpolated reject (both for chi2 and absolute distance)") AddOptionRTC(mergeCE, uint8_t, 1, "", 0, "Merge tracks accross the central electrode") -AddOptionRTC(retryRefit, int8_t, 1, "", 0, "Retry refit with seeding errors and without cluster rejection when fit fails (=2 means retry in same kernel, =1 for separate kernel") AddOptionRTC(enablePID, int8_t, 1, "", 0, "Enable PID response") AddOptionRTC(PID_useNsigma, int8_t, 1, "", 0, "Use nSigma instead of absolute distance in PID response") AddOptionRTC(adddEdxSubThresholdClusters, int8_t, 1, "", 0, "Add sub threshold clusters in TPC dEdx computation") diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 7b27bf4b105b5..89c6b160164d1 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -235,9 +235,6 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0); - if (param().rec.tpc.retryRefit == 1) { - runKernel(GetGridAuto(0), -1); - } runKernel(GetGridAuto(0)); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRefit, *mDebugFile); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index e89081aa40350..4f7fea2182738 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -402,8 +402,7 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) computePointerWithAlignment(mem, mSharedCount, mNMaxClusters); memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; - computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers - computePointerWithAlignment(mem, mRetryRefitIds, mNMaxTracks); // Reducing mNMaxTracks for mLoopData / mRetryRefitIds does not save memory, since the other parts are larger anyway + computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLooperCandidates, mNMaxLooperMatches); // MergeLoopers 1-3 diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index bf587454ab20e..2c6252c2a4fa2 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -67,7 +67,6 @@ class GPUTPCGMMerger : public GPUProcessor static constexpr const int32_t NSECTORS = GPUTPCGeometry::NSECTORS; //* N sectors struct memory { - GPUAtomic(uint32_t) nRetryRefit; GPUAtomic(uint32_t) nLoopData; GPUAtomic(uint32_t) nUnpackedTracks; GPUAtomic(uint32_t) nMergedTracks; @@ -129,7 +128,6 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } GPUhdi() uint32_t* TrackOrderProcess() const { return mTrackOrderProcess; } - GPUhdi() uint32_t* RetryRefitIds() const { return mRetryRefitIds; } GPUhdi() uint8_t* ClusterStateExt() const { return mClusterStateExt; } GPUhdi() GPUTPCGMLoopData* LoopData() const { return mLoopData; } GPUhdi() memory* Memory() const { return mMemory; } @@ -299,7 +297,6 @@ class GPUTPCGMMerger : public GPUProcessor gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRangeMemory = nullptr; // memory for border tracks gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRange[NSECTORS]; // memory for border tracks memory* mMemory = nullptr; - uint32_t* mRetryRefitIds = nullptr; GPUTPCGMLoopData* mLoopData = nullptr; }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 2a111b8ce89af..a7df0d279c08f 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -21,10 +21,9 @@ using namespace o2::gpu; template <> GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode) { - const int32_t iEnd = mode == -1 ? merger.Memory()->nRetryRefit : merger.NMergedTracks(); - GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, iEnd, { - const int32_t i = mode == -1 ? merger.RetryRefitIds()[ii] : mode ? merger.TrackOrderProcess()[ii] : ii; - GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger, mode == -1); + GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, merger.NMergedTracks(), { + const int32_t i = mode ? merger.TrackOrderProcess()[ii] : ii; + GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger); }); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 53e7f6c918309..4dde39c24cbc7 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -47,7 +47,7 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, int32_t attempt, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track) { static constexpr float kDeg2Rad = M_PI / 180.f; CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); @@ -91,7 +91,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); ResetCovariance(); - prop.SetSeedingErrors(!(refit && attempt == 0)); + prop.SetSeedingErrors(!(refit)); prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed prop.SetPropagateBzOnly(param.rec.fitPropagateBzOnly == -1 ? !finalFit : param.rec.fitPropagateBzOnly); prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger->GetConstantMem()->calibObjects.matLUT : nullptr); @@ -104,7 +104,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; - int32_t goodRows = 0; for (int32_t ihit = ihitStart; ihit >= 0 && ihit < maxN; ihit += wayDirection) { const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (clusters[ihit].sector < 18)); if (crossCE) { @@ -178,11 +177,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); } } - if (lastRow == 255 || CAMath::Abs((int32_t)lastRow - (int32_t)cluster.row) > 5 || lastSector != cluster.sector || (param.rec.tpc.trackFitRejectMode < 0 && -nMissed <= param.rec.tpc.trackFitRejectMode)) { - goodRows = 0; - } else { - goodRows++; - } if (retValProp == 0) { lastRow = cluster.row; lastSector = cluster.sector; @@ -227,17 +221,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; } else { int8_t rejectChi2 = 0; - if (attempt == 0) { - if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { - if (iWay == nWays - 3) { - rejectChi2 = GPUTPCGMPropagator::rejectInterFill; - } else if (iWay == nWays - 2) { - rejectChi2 = GPUTPCGMPropagator::rejectInterReject; - } else if (iWay == nWays - 1) { - rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; - } - } else { - rejectChi2 = allowChangeClusters && goodRows > 5; + if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { + if (iWay == nWays - 3) { + rejectChi2 = GPUTPCGMPropagator::rejectInterFill; + } else if (iWay == nWays - 2) { + rejectChi2 = GPUTPCGMPropagator::rejectInterReject; + } else if (iWay == nWays - 1) { + rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; } } @@ -904,7 +894,7 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const return ok; } -GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger, int32_t attempt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. +GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. { if (!track.OK()) { return; @@ -919,27 +909,10 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); CADEBUG(int32_t nTrackHitsOld = nTrackHits; float ptOld = t.QPt()); - bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, attempt, constants::MAX_SIN_PHI, track); + bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, constants::MAX_SIN_PHI, track); CADEBUG(printf("Finished Fit Track %d\n", iTrk)); CADEBUG(printf("OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", nTrackHitsOld, nTrackHits, NTolerated, nTrackHits + NTolerated, ptOld, t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); - if (!ok && attempt == 0 && merger->Param().rec.tpc.retryRefit) { - for (uint32_t i = 0; i < track.NClusters(); i++) { - merger->Clusters()[track.FirstClusterRef() + i].state &= GPUTPCGMMergedTrackHit::clustererAndSharedFlags; - } - CADEBUG(printf("Track rejected, marking for retry\n")); - if (merger->Param().rec.tpc.retryRefit == 2) { - nTrackHits = track.NClusters(); - NTolerated = 0; // Clusters not fit but tollerated for track length cut - t = track.Param(); - Alpha = track.Alpha(); - ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, 1, constants::MAX_SIN_PHI, track); - } else { - uint32_t nRefit = CAMath::AtomicAdd(&merger->Memory()->nRetryRefit, 1u); - merger->RetryRefitIds()[nRefit] = iTrk; - return; - } - } if (CAMath::Abs(t.QPt()) < 1.e-4f) { t.QPt() = 1.e-4f; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 51689753f1ca5..69f2cda7b69db 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -141,7 +141,7 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, int32_t attempt, float maxSinPhi, GPUTPCGMMergedTrack& track); + GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, float maxSinPhi, GPUTPCGMMergedTrack& track); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); @@ -200,7 +200,7 @@ class GPUTPCGMTrackParam } } - GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger, int32_t attempt); + GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger); GPUdi() void ConstrainSinPhi(float limit = constants::MAX_SIN_PHI) { From 4cf0e6b901c2140e19440d2ad8314903468dde47 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 4 Sep 2025 21:34:57 +0200 Subject: [PATCH 02/25] GPU TPC: Add possibility to run last way of TPC fit in separate kernel to rebuild track ion between --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + GPU/GPUTracking/Global/GPUChainTracking.cxx | 4 ++ .../Global/GPUChainTrackingMerger.cxx | 5 ++- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 4 +- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h | 2 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 44 ++++++++++++------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 5 +-- GPU/GPUTracking/kernels.cmake | 2 +- 8 files changed, 42 insertions(+), 25 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 8b08fcc648143..c27fc908c3f4e 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -138,6 +138,7 @@ AddOptionRTC(cfNoiseSuppressionEpsilon, uint8_t, 10, "", 0, "Cluster Finder: Dif AddOptionRTC(cfNoiseSuppressionEpsilonRelative, uint8_t, 76, "", 0, "Cluster Finder: Difference between peak and charge for the charge to count as a minima during noise suppression, relative as fraction of 255") AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 pads closes to the sector edge as edge cluster") AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") +AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits, <0: reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index dc7b23a375cd3..3350d12664c1a 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -261,6 +261,10 @@ bool GPUChainTracking::ValidateSettings() GPUError("Cannot do error interpolation with NWays < 3!"); return false; } + if (param().rec.tpc.rebuildTrackInFit && !param().rec.tpc.mergerInterpolateErrors) { + GPUError("Need error interpolation to rebuild tracks during fit"); + return false; + } if (param().continuousMaxTimeBin > (int32_t)GPUSettings::TPC_MAX_TF_TIME_BIN) { GPUError("configured max time bin exceeds 256 orbits"); return false; diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 89c6b160164d1..9e731b69f3826 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -234,7 +234,10 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) mOutputQueue.clear(); } - runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0); + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 0); + if (param().rec.tpc.rebuildTrackInFit) { + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); + } runKernel(GetGridAuto(0)); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRefit, *mDebugFile); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index a7df0d279c08f..22d8b159e9b74 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -19,11 +19,11 @@ using namespace o2::gpu; template <> -GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode) +GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode, int32_t rebuilt) { GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, merger.NMergedTracks(), { const int32_t i = mode ? merger.TrackOrderProcess()[ii] : ii; - GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger); + GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger, rebuilt); }); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h index 5d00451516aa8..0e3e4990ff04b 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h @@ -36,7 +36,7 @@ class GPUTPCGMMergerTrackFit : public GPUTPCGMMergerGeneral { public: template - GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger, int32_t mode); + GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger, int32_t mode, int32_t rebuilt); }; class GPUTPCGMMergerFollowLoopers : public GPUTPCGMMergerGeneral diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 4dde39c24cbc7..85ad986b9de74 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -47,7 +47,7 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { static constexpr float kDeg2Rad = M_PI / 180.f; CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); @@ -60,7 +60,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ prop.SetMaterialTPC(); prop.SetPolynomialField(¶m.polynomialField); prop.SetMaxSinPhi(maxSinPhi); - if (param.rec.tpc.mergerInterpolateErrors) { + if (param.rec.tpc.mergerInterpolateErrors && !rebuilt) { for (int32_t i = 0; i < N; i++) { interpolation.hit[i].errorY = -1; } @@ -70,12 +70,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const int32_t maxN = N; int32_t ihitStart = 0; float covYYUpd = 0.f; - float lastUpdateX = -1.f; - uint8_t lastRow = 255; - uint8_t lastSector = 255; float deltaZ = 0.f; - for (int32_t iWay = 0; iWay < nWays; iWay++) { + for (int32_t iWay = rebuilt ? nWays - 1 : 0; iWay < nWays; iWay++) { // DR: Unrolling has no performance improvement on GPU, why? int32_t nMissed = 0, nMissed2 = 0; float sumInvSqrtCharge = 0.f; int32_t nAvgCharge = 0; @@ -95,12 +92,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed prop.SetPropagateBzOnly(param.rec.fitPropagateBzOnly == -1 ? !finalFit : param.rec.fitPropagateBzOnly); prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger->GetConstantMem()->calibObjects.matLUT : nullptr); - prop.SetTrack(this, iWay ? prop.GetAlpha() : Alpha); + prop.SetTrack(this, iWay && !rebuilt ? prop.GetAlpha() : Alpha); ConstrainSinPhi(iWay == 0 ? 0.95f : constants::MAX_SIN_PHI_LOW); CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / kSectAngle) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); N = 0; - lastUpdateX = -1; + uint8_t lastRow = 255; + uint8_t lastSector = 255; + float lastUpdateX = -1; const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; @@ -110,9 +109,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ lastSector = clusters[ihit].sector; } - if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) { + if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); - if (finalOutInFit && !(clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject)) { + if (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl)) { + NTolerated++; + } + if (finalOutInFit && !(clusters[ihit].state & (GPUTPCGMMergedTrackHit::flagReject | GPUTPCGMMergedTrackHit::flagHighIncl))) { clusters[ihit].state |= GPUTPCGMMergedTrackHit::flagRejectErr; } continue; @@ -327,6 +329,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } else { deltaZ = 0.f; } + + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { + Alpha = prop.GetAlpha(); + if (ihitStart != 0) { + MarkClusters(clusters, 0, ihitStart - 1, 1, GPUTPCGMMergedTrackHit::flagHighIncl); + } + return true; + } } ConstrainSinPhi(); @@ -894,7 +904,7 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const return ok; } -GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. +GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger, bool rebuilt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. { if (!track.OK()) { return; @@ -908,21 +918,21 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - CADEBUG(int32_t nTrackHitsOld = nTrackHits; float ptOld = t.QPt()); - bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, constants::MAX_SIN_PHI, track); - CADEBUG(printf("Finished Fit Track %d\n", iTrk)); - CADEBUG(printf("OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", nTrackHitsOld, nTrackHits, NTolerated, nTrackHits + NTolerated, ptOld, t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); + bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, constants::MAX_SIN_PHI, track, rebuilt); + CADEBUG(if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); if (CAMath::Abs(t.QPt()) < 1.e-4f) { - t.QPt() = 1.e-4f; + t.QPt() = CAMath::Copysign(1.e-4f, t.QPt()); } CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger->MergedTracks()[iTrk].Looper()); }); track.SetOK(ok); - track.SetNClustersFitted(nTrackHits); track.Param() = t; track.Alpha() = Alpha; + if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) { + track.SetNClustersFitted(nTrackHits); + } // if (track.OK()) merger->DebugRefitMergedTrack(track); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 69f2cda7b69db..a6158ca915398 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -141,7 +141,8 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, float maxSinPhi, GPUTPCGMMergedTrack& track); + GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, float maxSinPhi, GPUTPCGMMergedTrack& track, bool rebuilt); + GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); @@ -200,8 +201,6 @@ class GPUTPCGMTrackParam } } - GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger); - GPUdi() void ConstrainSinPhi(float limit = constants::MAX_SIN_PHI) { if (mP[2] > limit) { diff --git a/GPU/GPUTracking/kernels.cmake b/GPU/GPUTracking/kernels.cmake index 2176ea2dc3804..93b5dfdf6158c 100644 --- a/GPU/GPUTracking/kernels.cmake +++ b/GPU/GPUTracking/kernels.cmake @@ -51,7 +51,7 @@ o2_gpu_add_kernel("GPUTPCGlobalDebugSortKernels, mergedTracks2" "= TPC o2_gpu_add_kernel("GPUTPCGlobalDebugSortKernels, borderTracks" "= TPCMERGER" NO int8_t parameter) o2_gpu_add_kernel("GPUTPCCreateOccupancyMap, fill" "= TPCOCCUPANCY" LB GPUTPCClusterOccupancyMapBin* map) o2_gpu_add_kernel("GPUTPCCreateOccupancyMap, fold" "= TPCOCCUPANCY" LB GPUTPCClusterOccupancyMapBin* map uint32_t* output) -o2_gpu_add_kernel("GPUTPCGMMergerTrackFit" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT TPCDEDX" LB int32_t mode) +o2_gpu_add_kernel("GPUTPCGMMergerTrackFit" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT TPCDEDX" LB int32_t mode int32_t rebuilt) o2_gpu_add_kernel("GPUTPCGMMergerFollowLoopers" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT" LB) o2_gpu_add_kernel("GPUTPCGMMergerUnpackResetIds" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iSector) o2_gpu_add_kernel("GPUTPCGMMergerSectorRefit" "GPUTPCGMMergerGPU TPCMERGER MATLUT" LB int32_t iSector) From e661eb85bb768e221c1d69b2a4d34e4390910aae Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 23 Sep 2025 17:58:33 +0200 Subject: [PATCH 03/25] GPU TPC: Search hit closest to interpolated point in track rebuilding --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + .../Global/GPUChainTrackingMerger.cxx | 26 ++-- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 5 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 5 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 131 ++++++++++++++++-- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 2 +- GPU/GPUTracking/SectorTracker/GPUTPCTracker.h | 1 + .../GPUTPCTrackletConstructor.cxx | 4 +- 8 files changed, 149 insertions(+), 26 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index c27fc908c3f4e..8d17c35c4c3e4 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -139,6 +139,7 @@ AddOptionRTC(cfNoiseSuppressionEpsilonRelative, uint8_t, 76, "", 0, "Cluster Fin AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 pads closes to the sector edge as edge cluster") AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") +AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits, <0: reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 9e731b69f3826..2dfdd4978ec89 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -96,7 +96,6 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) uint32_t numBlocks = (!mRec->IsGPU() || doGPU) ? BlockCount() : 1; GPUTPCGMMerger& Merger = processors()->tpcMerger; GPUTPCGMMerger& MergerShadow = doGPU ? processorsShadow()->tpcMerger : Merger; - GPUTPCGMMerger& MergerShadowAll = doGPU ? processorsShadow()->tpcMerger : Merger; const int32_t outputStream = OutputStream(); if (GetProcessingSettings().debugLevel >= 2) { GPUInfo("Running TPC Merger"); @@ -142,7 +141,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) // Merge within Sectors runKernel(GetGridAuto(0, deviceType), false); - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), NSECTORS * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridAuto(0, deviceType)); RunTPCTrackingMerger_MergeBorderTracks(GPUTPCGMMerger::mergeModes::mergeWithinSector, deviceType); RunTPCTrackingMerger_Resolve(0, 1, deviceType); @@ -150,23 +149,23 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) // Merge between sectors - transport to the middle of the sector and rotate vertically to the border on the left / right runKernel(GetGridAuto(0, deviceType), false); - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), 2, 3, 0); RunTPCTrackingMerger_MergeBorderTracks(GPUTPCGMMerger::mergeModes::mergeBetweenSector | GPUTPCGMMerger::mergeModes::mergeAtMidRow, deviceType); RunTPCTrackingMerger_Resolve(0, 1, deviceType); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadow.TmpCounter())); // Merge between sectors - transport to the left / right edge of the sector and rotate horizontally - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), 0, 1, 0); RunTPCTrackingMerger_MergeBorderTracks(GPUTPCGMMerger::mergeModes::mergeBetweenSector, deviceType); RunTPCTrackingMerger_Resolve(0, 1, deviceType); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadow.TmpCounter())); // Merge between sectors - use original track param - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), 0, 1, 1); RunTPCTrackingMerger_MergeBorderTracks(GPUTPCGMMerger::mergeModes::mergeBetweenSector | GPUTPCGMMerger::mergeModes::mergeWithOriginalParameters, deviceType); RunTPCTrackingMerger_Resolve(0, 1, deviceType); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingMatching, doGPU, Merger, &GPUTPCGMMerger::DumpMergedBetweenSectors, *mDebugFile); - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridAuto(0, deviceType)); if (GetProcessingSettings().mergerSanityCheck) { @@ -208,8 +207,8 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) if (maxId > Merger.NMaxClusters()) { throw std::runtime_error("mNMaxClusters too small"); } - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.SharedCount(), maxId * sizeof(*MergerShadowAll.SharedCount())); - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.ClusterAttachment(), maxId * sizeof(*MergerShadowAll.ClusterAttachment())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.SharedCount(), maxId * sizeof(*MergerShadow.SharedCount())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterAttachment(), maxId * sizeof(*MergerShadow.ClusterAttachment())); runKernel(GetGridAuto(0, deviceType)); CondWaitEvent(waitForTransfer, &mEvents->single); runKernel(GetGridAuto(0, deviceType)); @@ -234,6 +233,9 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) mOutputQueue.clear(); } + if (param().rec.tpc.rebuildTrackInFit) { + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterCandidates(), Merger.NMergedTracks() * GPUTPCGeometry::NROWS * param().rec.tpc.rebuildTrackInFitClusterCandidates * sizeof(*MergerShadow.ClusterCandidates())); + } runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 0); if (param().rec.tpc.rebuildTrackInFit) { runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); @@ -268,13 +270,13 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) throw std::runtime_error("QA Scratch buffer exceeded"); } } - GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracks(), MergerShadowAll.MergedTracks(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracks()), outputStream, 0, nullptr, waitEvent); + GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracks(), MergerShadow.MergedTracks(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracks()), outputStream, 0, nullptr, waitEvent); waitEvent = nullptr; if (param().dodEdxEnabled) { - GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracksdEdx(), MergerShadowAll.MergedTracksdEdx(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracksdEdx()), outputStream, 0); + GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracksdEdx(), MergerShadow.MergedTracksdEdx(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracksdEdx()), outputStream, 0); } - GPUMemCpy(RecoStep::TPCMerging, Merger.Clusters(), MergerShadowAll.Clusters(), Merger.NMergedTrackClusters() * sizeof(*Merger.Clusters()), outputStream, 0); - GPUMemCpy(RecoStep::TPCMerging, Merger.ClusterAttachment(), MergerShadowAll.ClusterAttachment(), Merger.NMaxClusters() * sizeof(*Merger.ClusterAttachment()), outputStream, 0); + GPUMemCpy(RecoStep::TPCMerging, Merger.Clusters(), MergerShadow.Clusters(), Merger.NMergedTrackClusters() * sizeof(*Merger.Clusters()), outputStream, 0); + GPUMemCpy(RecoStep::TPCMerging, Merger.ClusterAttachment(), MergerShadow.ClusterAttachment(), Merger.NMaxClusters() * sizeof(*Merger.ClusterAttachment()), outputStream, 0); } if (GetProcessingSettings().outputSharedClusterMap) { TransferMemoryResourceLinkToHost(RecoStep::TPCMerging, Merger.MemoryResOutputState(), outputStream, nullptr, waitEvent); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 4f7fea2182738..05af67834bf68 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -403,6 +403,7 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway + computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * GPUTPCGeometry::NROWS * Param().rec.tpc.rebuildTrackInFitClusterCandidates); memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLooperCandidates, mNMaxLooperMatches); // MergeLoopers 1-3 @@ -1655,7 +1656,7 @@ GPUd() void GPUTPCGMMerger::CollectMergedTracks(int32_t nBlocks, int32_t nThread const GPUTPCTracker& trk = GetConstantMem()->tpcTrackers[t->Sector()]; const GPUTPCHitId& ic = trk.TrackHits()[t->OrigTrack()->FirstHitID() + i]; uint32_t id = trk.Data().ClusterDataIndex(trk.Data().Row(ic.RowIndex()), ic.HitIndex()) + GetConstantMem()->ioPtrs.clustersNative->clusterOffset[t->Sector()][0]; - *c2 = trackCluster{id, (uint8_t)ic.RowIndex(), t->Sector()}; + *c2 = trackCluster{.id = id, .row = (uint8_t)ic.RowIndex(), .sector = t->Sector(), .error = 0.f}; } nHits += nTrackHits; } @@ -1963,7 +1964,7 @@ GPUd() void GPUTPCGMMerger::MergeLoopersInit(int32_t nBlocks, int32_t nThreads, for (uint32_t k = 0;k < trk.NClusters();k++) { float xx, yy, zz; const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[mClusters[trk.FirstClusterRef() + k].num]; - GetConstantMem()->calibObjects.fastTransformHelper->Transform(mClusters[trk.FirstClusterRef() + k].sector, mClusters[trk.FirstClusterRef() + k].row, cl.getPad(), cl.getTime(), xx, yy, zz, p.GetTOffset()); + GetConstantMem()->calibObjects.fastTransform->Transform(mClusters[trk.FirstClusterRef() + k].sector, mClusters[trk.FirstClusterRef() + k].row, cl.getPad(), cl.getTime(), xx, yy, zz, p.GetTOffset()); float sa2, ca2; CAMath::SinCos(Param().Alpha(mClusters[trk.FirstClusterRef() + k].sector), sa2, ca2); float cx = ca2 * xx - sa2 * yy; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 2c6252c2a4fa2..6be66bcc9447e 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -78,10 +78,11 @@ class GPUTPCGMMerger : public GPUProcessor GPUAtomic(uint32_t) nLooperMatchCandidates; }; - struct trackCluster { + struct trackCluster { // TODO: Reduce size of this struct! uint32_t id; uint8_t row; uint8_t sector; + float error; }; struct tmpSort { @@ -125,6 +126,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() uint32_t NMergedTrackClusters() const { return mMemory->nMergedTrackClusters; } GPUhdi() const GPUTPCGMMergedTrackHit* Clusters() const { return mClusters; } GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return (mClusters); } + GPUhdi() trackCluster* ClusterCandidates() { return (mClusterCandidates); } GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } GPUhdi() uint32_t* TrackOrderProcess() const { return mTrackOrderProcess; } @@ -272,6 +274,7 @@ class GPUTPCGMMerger : public GPUProcessor int32_t mNClusters = 0; // Total number of incoming clusters (from sector tracks) GPUTPCGMMergedTrack* mMergedTracks = nullptr; //* array of output merged tracks + trackCluster* mClusterCandidates = nullptr; GPUdEdxInfo* mMergedTracksdEdx = nullptr; //* dEdx information GPUdEdxInfo* mMergedTracksdEdxAlt = nullptr; //* dEdx alternative information GPUTPCGMSectorTrack* mSectorTrackInfos = nullptr; //* additional information for sector tracks diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 85ad986b9de74..bf28fe9c75d3f 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -47,9 +47,10 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { static constexpr float kDeg2Rad = M_PI / 180.f; + static constexpr float maxSinPhi = constants::MAX_SIN_PHI; CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); const GPUParam& GPUrestrict() param = merger->Param(); @@ -72,9 +73,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ float covYYUpd = 0.f; float deltaZ = 0.f; - for (int32_t iWay = rebuilt ? nWays - 1 : 0; iWay < nWays; iWay++) { // DR: Unrolling has no performance improvement on GPU, why? + for (int32_t iWay = rebuilt ? nWays - 1 : 0; iWay < nWays; iWay++) { // TODO DR: Unrolling has no performance improvement on GPU, why? int32_t nMissed = 0, nMissed2 = 0; - float sumInvSqrtCharge = 0.f; + float sumInvSqrtCharge = 0.f; // TODO: Compute in first iteration and store! int32_t nAvgCharge = 0; if (iWay && (iWay & 1) == 0) { @@ -135,7 +136,26 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ // CADEBUG(if ((uint32_t)merger->GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger->GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); // clang-format on - if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, allowChangeClusters) == -1) { + if (rebuilt && merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + clusters[ihit].row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id > 0) { + while (true) { + if (!(ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector)) { + break; + } + } + const auto& interHit = merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + clusters[ihit].row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; + if (interHit.id == 1) { + nMissed++; + nMissed2++; + continue; + } + const ClusterNative& GPUrestrict() cl = merger -> GetConstantMem()->ioPtrs.clustersNative->clustersLinear[interHit.id - 2]; + merger->GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); + if (interHit.id - 2 == clusters[ihit].num) { + clusterState = clusters[ihit].state; + } else { + clusterState = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; + } + } else if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { nMissed++; nMissed2++; continue; @@ -155,7 +175,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ // clang-format off CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); // clang-format on - if (allowChangeClusters && lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { + if (!param.rec.tpc.rebuildTrackInFit && allowChangeClusters && lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; dodEdx = AttachClustersPropagate(merger, cluster.sector, lastRow, cluster.row, iTrk, track.Leg() == 0, prop, inFlyDirection, constants::MAX_SIN_PHI, dodEdx); @@ -200,7 +220,102 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } float uncorrectedY = -1e6f; - if (allowChangeClusters) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { + const GPUTPCTracker& GPUrestrict() tracker = *(merger -> GetConstantMem()->tpcTrackers + cluster.sector); + const GPUTPCRow& GPUrestrict() row = tracker.Row(cluster.row); + GPUglobalref() const cahit2* hits = tracker.HitData(row); + GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); + const auto& inter = interpolation.hit[ihit]; + if (row.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + const float zOffset = merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin); + const float y0 = row.Grid().YMin(); + const float stepY = row.HstepY(); + const float z0 = row.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway + const float stepZ = row.HstepZ(); + int32_t bin, ny, nz; + + float err2Y, err2Z; + param.GetClusterErrors2(cluster.sector, cluster.row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + + const float Iz0 = inter.posY - mP[0]; + const float Iz1 = inter.posZ + deltaZ - mP[1]; + const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); + const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); + const float Ik00 = mC[0] * Iw0; + const float Ik11 = mC[2] * Iw2; + const float ImP0 = mP[0] + Ik00 * Iz0; + const float ImP1 = mP[1] + Ik11 * Iz1; + const float ImC0 = mC[0] - Ik00 * mC[0]; + const float ImC2 = mC[2] - Ik11 * mC[2]; + + float uncorrectedZ; + merger->GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, ImP0, ImP1, uncorrectedY, uncorrectedZ); + + int32_t nCandidates = 0; + if (CAMath::Abs(uncorrectedY) <= row.getTPCMaxY()) { + const float kFactor = tracker.GetChiSeedFactor(); + const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? + const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(mC[2]))); + const float tubeY = CAMath::Sqrt(sy2); + const float tubeZ = CAMath::Sqrt(sz2); + row.Grid().GetBinArea(uncorrectedY, uncorrectedZ + zOffset, tubeY, tubeZ, bin, ny, nz); + + const int32_t nBinsY = row.Grid().Ny(); + const int32_t idOffset = tracker.Data().ClusterIdOffset(); + const int32_t* ids = &(tracker.Data().ClusterDataIndex()[row.HitNumberOffset()]); + for (int32_t k = 0; k <= nz; k++) { + const int32_t mybin = bin + k * nBinsY; + const uint32_t hitFst = firsthit[mybin]; + const uint32_t hitLst = firsthit[mybin + ny + 1]; + for (uint32_t ih = hitFst; ih < hitLst; ih++) { + const cahit2 hh = hits[ih]; + const float y = y0 + hh.x * stepY; + const float z = z0 + hh.y * stepZ; + const float dy = y - uncorrectedY; + const float dz = z - uncorrectedZ; + + if (dy * dy < sy2 && dz * dz < sz2) { + float err2YA, err2ZA; + const ClusterNative& GPUrestrict() cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[idOffset + ids[ih]]; + const auto clflags = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; + const float time = cl.getTime(); + const float invSqrtCharge = CAMath::InvSqrt(cl.qMax); + const float invCharge = 1.f / cl.qMax; + float invAvgCharge = (sumInvSqrtCharge + invSqrtCharge) / (nAvgCharge + 1); + invAvgCharge *= invAvgCharge; + + prop.GetErr2(err2YA, err2ZA, merger->Param(), mP[1], cluster.row, clflags, cluster.sector, time, invAvgCharge, invCharge); + const float Jw0 = 1.f / (ImC0 + err2YA); + const float Jw2 = 1.f / (ImC2 + err2ZA); + const float chi2Y = Jw0 * dy * dy; + const float chi2Z = Jw2 * dz * dz; + bool ok = !prop.RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clflags); + float err = dy * dy + dz * dz; + if (ok) { + int32_t insert = nCandidates; + for (int32_t c = 0; c < nCandidates; c++) { + if (err < merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c].error) { + insert = c; + break; + } + } + if (insert < param.rec.tpc.rebuildTrackInFitClusterCandidates) { + for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { + merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; + } + merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = cluster.row, .sector = cluster.sector, .error = err}; + nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); + } + } + } + } + } + } + if (nCandidates == 0) { + merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; + } + } + } else if (allowChangeClusters) { uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); } @@ -320,7 +435,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ break; // bad chi2 for the whole track, stop the fit } } - if (finalOutInFit && !(merger->Param().rec.tpc.disableRefitAttachment & 4) && lastRow != 255 && lastSector != 255) { + if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255 && lastSector != 255) { StoreLoopPropagation(merger, lastSector, lastRow, iTrk, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastRow %d row %d out %d\n", iTrk, (int)lastRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } @@ -918,7 +1033,7 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, constants::MAX_SIN_PHI, track, rebuilt); + bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, track, rebuilt); CADEBUG(if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); if (CAMath::Abs(t.QPt()) < 1.e-4f) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index a6158ca915398..ce4ab8f463d76 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -141,7 +141,7 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, float maxSinPhi, GPUTPCGMMergedTrack& track, bool rebuilt); + GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h b/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h index cb0c8d4a76fdb..7c9047202417b 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h @@ -167,6 +167,7 @@ class GPUTPCTracker : public GPUProcessor } return ((int32_t)weight); } + GPUdi() float GetChiSeedFactor() const { return Param().rec.tpc.hitPickUpFactor * Param().rec.tpc.hitPickUpFactor * 3.5f * 3.5f; } GPUd() void MaximizeHitWeight(const GPUTPCRow& row, int32_t hitIndex, int32_t weight) { mData.MaximizeHitWeight(row, hitIndex, weight); } GPUd() void SetHitWeight(const GPUTPCRow& row, int32_t hitIndex, int32_t weight) { mData.SetHitWeight(row, hitIndex, weight); } GPUd() int32_t HitWeight(const GPUTPCRow& row, int32_t hitIndex) const { return mData.HitWeight(row, hitIndex); } diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx index 33a3264a87ab3..86979c29006ae 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx @@ -199,7 +199,7 @@ GPUdic(2, 1) void GPUTPCTrackletConstructor::UpdateTracklet(int32_t /*nBlocks*/, const float sErr2 = tracker.Param().GetSystematicClusterErrorIFC2(x, tParam.GetY(), tParam.GetZ(), tracker.ISector() >= 18); err2Y += sErr2; err2Z += sErr2; - const float kFactor = tracker.Param().rec.tpc.hitPickUpFactor * tracker.Param().rec.tpc.hitPickUpFactor * 3.5f * 3.5f; + const float kFactor = tracker.GetChiSeedFactor(); float sy2 = kFactor * (tParam.Err2Y() + err2Y); float sz2 = kFactor * (tParam.Err2Z() + err2Z); if (sy2 > tracker.Param().rec.tpc.hitSearchArea2) { @@ -307,7 +307,7 @@ GPUdic(2, 1) void GPUTPCTrackletConstructor::UpdateTracklet(int32_t /*nBlocks*/, const float kFactor = tracker.Param().rec.tpc.hitPickUpFactor * tracker.Param().rec.tpc.hitPickUpFactor * 7.0f * 7.0f; const float maxWindow2 = tracker.Param().rec.tpc.hitSearchArea2; const float sy2 = CAMath::Min(maxWindow2, kFactor * (tParam.Err2Y() + err2Y)); - const float sz2 = CAMath::Min(maxWindow2, kFactor * (tParam.Err2Z() + err2Z)); + const float sz2 = CAMath::Min(maxWindow2, kFactor * (tParam.Err2Z() + err2Z)); // TODO: Double check if the factors are enough, errors could be larger with flags int32_t bin, ny, nz; row.Grid().GetBinArea(yUncorrected, zUncorrected + tParam.ZOffset(), CAMath::Sqrt(sy2), CAMath::Sqrt(sz2), bin, ny, nz); From 0228265ee385e0074bf32f11d9db822205d01a50 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 5 Oct 2025 18:31:00 +0200 Subject: [PATCH 04/25] GPU TPC: Fix index for hit interpolation in rows with multiple clusters --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index bf28fe9c75d3f..0ee2ac74cb561 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -219,13 +219,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } } + auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { const GPUTPCTracker& GPUrestrict() tracker = *(merger -> GetConstantMem()->tpcTrackers + cluster.sector); const GPUTPCRow& GPUrestrict() row = tracker.Row(cluster.row); GPUglobalref() const cahit2* hits = tracker.HitData(row); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); - const auto& inter = interpolation.hit[ihit]; if (row.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { const float zOffset = merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin); const float y0 = row.Grid().YMin(); @@ -357,10 +357,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); if (rejectChi2 >= GPUTPCGMPropagator::rejectInterFill) { - if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && interpolation.hit[ihit].errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { rejectChi2 = GPUTPCGMPropagator::rejectDirect; } else { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &interpolation.hit[ihit], err2Y, err2Z, deltaZ); + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &inter, err2Y, err2Z, deltaZ); } } @@ -370,7 +370,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ retValUpd = prop.Update(yy, zz, cluster.row, param, clusterState, rejectChi2, refit, err2Y, err2Z); } GPUCA_DEBUG_STREAMER_CHECK(if (o2::utils::DebugStreamer::checkStream(o2::utils::StreamFlags::streamUpdateTrack, iTrk)) { - merger->DebugStreamerUpdate(iTrk, ihit, xx, yy, zz, cluster, merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num], *this, prop, interpolation.hit[ihit], rejectChi2, refit, retValUpd, sumInvSqrtCharge / nAvgCharge * sumInvSqrtCharge / nAvgCharge, yy, zz, clusterState, retValInt, err2Y, err2Z); + merger->DebugStreamerUpdate(iTrk, ihit, xx, yy, zz, cluster, merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num], *this, prop, inter, rejectChi2, refit, retValUpd, sumInvSqrtCharge / nAvgCharge * sumInvSqrtCharge / nAvgCharge, yy, zz, clusterState, retValInt, err2Y, err2Z); }); } // clang-format off From 2a2d02e61857c2ee79b8ff16e6515aafe616ac05 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 29 Sep 2025 22:01:44 +0200 Subject: [PATCH 05/25] GPU TPC: Implement ambiguity solving for rebuild tracks cluster association --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 3 +- .../Definitions/Parameters/GPUParameters.csv | 5 + .../GPUChainTrackingDebugAndProfiling.cxx | 6 +- .../Global/GPUChainTrackingMerger.cxx | 22 +- GPU/GPUTracking/Global/GPUErrorCodes.h | 20 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 216 ++++++++++++++++-- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 27 ++- GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx | 4 +- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 31 +++ GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h | 13 ++ GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 36 +-- GPU/GPUTracking/SectorTracker/GPUTPCTracker.h | 5 +- .../Standalone/Benchmark/standalone.cxx | 2 +- GPU/GPUTracking/kernels.cmake | 5 + GPU/GPUTracking/qa/GPUQA.cxx | 2 +- 15 files changed, 318 insertions(+), 79 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 8d17c35c4c3e4..db541b0270b49 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -140,12 +140,13 @@ AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") +AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits, <0: reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking (prolong tracks to adjacent sectors to find short segments)") -AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit (1: attachment, 2: propagation, 4: loop following, 8: mirroring)") +AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit (1: attachment, 2: propagation, 4: loop following)") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression (0 = no, 1 = strategy A, 2 = strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits (1=truncate charge/width LSB, 2=differences, 4=track-model)") diff --git a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv index ef215ba5ca870..c50fe3a56c8e9 100644 --- a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv +++ b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv @@ -52,6 +52,11 @@ GPUTPCGMMergerPrepareForFit_step2,256,,256,256,,,,,,256,256,256,,,,256 GPUTPCGMMergerFinalize_step0,256,,,256,,,,,,,,,,,,256 GPUTPCGMMergerFinalize_step1,256,,,256,,,,,,,,,,,,256 GPUTPCGMMergerFinalize_step2,256,,,256,,,,,,,,,,,,256 +GPUTPCGMMergerHitWeights_prepare,256,,256,256,,,,,,256,256,256,,,,256 +GPUTPCGMMergerHitWeights_compute,256,,256,256,,,,,,256,256,256,,,,256 +GPUTPCGMMergerHitWeights_resolve1,256,,256,256,,,,,,256,256,256,,,,256 +GPUTPCGMMergerHitWeights_resolve2,256,,256,256,,,,,,256,256,256,,,,256 +GPUTPCGMMergerHitWeights_resolveShared,256,,256,256,,,,,,256,256,256,,,,256 GPUTPCGMMergerMergeLoopers_step0,256,,,,,,,,,,,,,,,256 GPUTPCGMMergerMergeLoopers_step1,256,,,,,,,,,,,,,,,256 GPUTPCGMMergerMergeLoopers_step2,256,,,,,,,,,,,,,,,256 diff --git a/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx b/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx index 8f200d2c57a6d..7d0fe573ec7c2 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx @@ -187,8 +187,8 @@ void GPUChainTracking::PrintMemoryRelations() GPUInfo("MEMREL SectorTrackHits NCl %d NTrkH %d", processors()->tpcTrackers[i].NHitsTotal(), *processors()->tpcTrackers[i].NTrackHits()); } if (processors()->tpcMerger.Memory()) { - GPUInfo("MEMREL Tracks NCl %d NTrk %d", processors()->tpcMerger.NMaxClusters(), processors()->tpcMerger.NMergedTracks()); - GPUInfo("MEMREL TrackHitss NCl %d NTrkH %d", processors()->tpcMerger.NMaxClusters(), processors()->tpcMerger.NMergedTrackClusters()); + GPUInfo("MEMREL Tracks NCl %d NTrk %d", processors()->tpcMerger.NClusters(), processors()->tpcMerger.NMergedTracks()); + GPUInfo("MEMREL TrackHitss NCl %d NTrkH %d", processors()->tpcMerger.NClusters(), processors()->tpcMerger.NMergedTrackClusters()); } } @@ -217,7 +217,7 @@ void GPUChainTracking::PrintKernelDebugOutput() void GPUChainTracking::PrintOutputStat() { int32_t nTracks = 0, nAttachedClusters = 0, nAttachedClustersFitted = 0, nAdjacentClusters = 0; - uint32_t nCls = GetProcessingSettings().doublePipeline ? mIOPtrs.clustersNative->nClustersTotal : processors()->tpcMerger.NMaxClusters(); + uint32_t nCls = GetProcessingSettings().doublePipeline ? mIOPtrs.clustersNative->nClustersTotal : processors()->tpcMerger.NClusters(); if (GetProcessingSettings().createO2Output > 1) { nTracks = mIOPtrs.nOutputTracksTPCO2; nAttachedClusters = mIOPtrs.nMergedTrackHits; diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 2dfdd4978ec89..60dd8f591e0f5 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -203,12 +203,8 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) Merger.CheckCollectedTracks(); } - uint32_t maxId = Merger.NMaxClusters(); - if (maxId > Merger.NMaxClusters()) { - throw std::runtime_error("mNMaxClusters too small"); - } - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.SharedCount(), maxId * sizeof(*MergerShadow.SharedCount())); - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterAttachment(), maxId * sizeof(*MergerShadow.ClusterAttachment())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.SharedCount(), Merger.NClusters() * sizeof(*MergerShadow.SharedCount())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterAttachment(), Merger.NClusters() * sizeof(*MergerShadow.ClusterAttachment())); runKernel(GetGridAuto(0, deviceType)); CondWaitEvent(waitForTransfer, &mEvents->single); runKernel(GetGridAuto(0, deviceType)); @@ -238,6 +234,16 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 0); if (param().rec.tpc.rebuildTrackInFit) { + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.HitWeights(), Merger.NClusters() * sizeof(*MergerShadow.HitWeights())); + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); + for (int32_t i = 0; i < param().rec.tpc.rebuildTrackInFitClusterCandidates; i++) { + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), i); + if (i + 1 < param().rec.tpc.rebuildTrackInFitClusterCandidates) { + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), i); + } + } + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); } runKernel(GetGridAuto(0)); @@ -276,7 +282,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracksdEdx(), MergerShadow.MergedTracksdEdx(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracksdEdx()), outputStream, 0); } GPUMemCpy(RecoStep::TPCMerging, Merger.Clusters(), MergerShadow.Clusters(), Merger.NMergedTrackClusters() * sizeof(*Merger.Clusters()), outputStream, 0); - GPUMemCpy(RecoStep::TPCMerging, Merger.ClusterAttachment(), MergerShadow.ClusterAttachment(), Merger.NMaxClusters() * sizeof(*Merger.ClusterAttachment()), outputStream, 0); + GPUMemCpy(RecoStep::TPCMerging, Merger.ClusterAttachment(), MergerShadow.ClusterAttachment(), Merger.NClusters() * sizeof(*Merger.ClusterAttachment()), outputStream, 0); } if (GetProcessingSettings().outputSharedClusterMap) { TransferMemoryResourceLinkToHost(RecoStep::TPCMerging, Merger.MemoryResOutputState(), outputStream, nullptr, waitEvent); @@ -366,7 +372,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } if (GetProcessingSettings().debugLevel >= 2) { - GPUInfo("TPC Merger Finished (output clusters %d / input clusters %d)", Merger.NMergedTrackClusters(), Merger.NClusters()); + GPUInfo("TPC Merger Finished (output clusters %d / input clusters %d)", Merger.NMergedTrackClusters(), Merger.NSectorHits()); } return 0; } diff --git a/GPU/GPUTracking/Global/GPUErrorCodes.h b/GPU/GPUTracking/Global/GPUErrorCodes.h index a4921f478b107..2080548c7775c 100644 --- a/GPU/GPUTracking/Global/GPUErrorCodes.h +++ b/GPU/GPUTracking/Global/GPUErrorCodes.h @@ -33,16 +33,16 @@ GPUCA_ERROR_CODE(13, ERROR_SECTORDATA_HITINROW_OVERFLOW, SectorRow, Value, Max) GPUCA_ERROR_CODE(14, ERROR_SECTORDATA_BIN_OVERFLOW, SectorRow, Value, Max) GPUCA_ERROR_CODE(15, ERROR_SECTORDATA_Z_OVERFLOW, Sector, Value) GPUCA_ERROR_CODE(16, ERROR_MERGER_HIT_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(17, ERROR_MERGER_TRACK_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(18, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(19, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(20, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max) -GPUCA_ERROR_CODE(21, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max) -GPUCA_ERROR_CODE(22, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(23, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(24, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(25, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER - +GPUCA_ERROR_CODE(17, ERROR_MERGER_REBUILD_HIT_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(18, ERROR_MERGER_TRACK_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(19, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(20, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(21, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max) +GPUCA_ERROR_CODE(22, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max) +GPUCA_ERROR_CODE(23, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(24, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(25, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(25, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER GPUCA_ERROR_CODE(26, ERROR_TPCZS_INVALID_ROW, SectorRow) // Data from invalid row is skipped GPUCA_ERROR_CODE(27, ERROR_TPCZS_INVALID_NADC, SectorCRU, SamplesInPage, SamplesWritten) // Invalid number of ADC samples in header, existing samples were decoded GPUCA_ERROR_CODE(28, ERROR_TPCZS_INCOMPLETE_HBF, SectorCRU, PacketCount, NextPacketCount) // Part of HBF is missing, decoding incomplete diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 05af67834bf68..49d041408cb00 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -398,12 +398,17 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) computePointerWithAlignment(mem, mTrackIDs, GPUTPCGeometry::NSECTORS * mNMaxSingleSectorTracks); // UnpackResetIds - RefitSectorTracks - UnpackSectorGlobal memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; - computePointerWithAlignment(mem, mTrackSort, mNMaxTracks); // PrepareForFit0 - SortTracksQPt - PrepareForFit1 - PrepareForFit1 / Finalize0 - Finalize2 - computePointerWithAlignment(mem, mSharedCount, mNMaxClusters); + computePointerWithAlignment(mem, mSharedCount, mNClusters); // PrepareForFit0 - SortTracksQPt - PrepareForFit1 - PrepareForFit1 / Finalize0 - Finalize2 + computePointerWithAlignment(mem, mTrackSort, mNMaxTracks); memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; - computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway - computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * GPUTPCGeometry::NROWS * Param().rec.tpc.rebuildTrackInFitClusterCandidates); + computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway + if (mRec->GetParam().rec.tpc.rebuildTrackInFit) { + computePointerWithAlignment(mem, mSharedCount2, mNClusters); + computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * mNMaxTracks * GPUTPCGeometry::NROWS * Param().rec.tpc.rebuildTrackInFitClusterCandidates); + computePointerWithAlignment(mem, mTrackRebuildHelper, mNMaxTracks); + computePointerWithAlignment(mem, mHitWeights, mNClusters); + } memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLooperCandidates, mNMaxLooperMatches); // MergeLoopers 1-3 @@ -437,14 +442,14 @@ void* GPUTPCGMMerger::SetPointersOutput(void* mem) } } computePointerWithAlignment(mem, mClusters, mNMaxMergedTrackClusters); - computePointerWithAlignment(mem, mClusterAttachment, mNMaxClusters); + computePointerWithAlignment(mem, mClusterAttachment, mNClusters); return mem; } void* GPUTPCGMMerger::SetPointersOutputState(void* mem) { if ((mRec->GetRecoSteps() & gpudatatypes::RecoStep::Refit) || mRec->GetProcessingSettings().outputSharedClusterMap) { - computePointerWithAlignment(mem, mClusterStateExt, mNMaxClusters); + computePointerWithAlignment(mem, mClusterStateExt, mNClusters); } else { mClusterStateExt = nullptr; } @@ -497,33 +502,33 @@ void GPUTPCGMMerger::RegisterMemoryAllocation() void GPUTPCGMMerger::SetMaxData(const GPUTrackingInOutPointers& io) { mNTotalSectorTracks = 0; - mNClusters = 0; + mNSectorHits = 0; mNMaxSingleSectorTracks = 0; for (int32_t iSector = 0; iSector < NSECTORS; iSector++) { uint32_t ntrk = *mRec->GetConstantMem().tpcTrackers[iSector].NTracks(); mNTotalSectorTracks += ntrk; - mNClusters += *mRec->GetConstantMem().tpcTrackers[iSector].NTrackHits(); + mNSectorHits += *mRec->GetConstantMem().tpcTrackers[iSector].NTrackHits(); if (mNMaxSingleSectorTracks < ntrk) { mNMaxSingleSectorTracks = ntrk; } } - mNMaxMergedTrackClusters = mRec->MemoryScalers()->NTPCMergedTrackHits(mNClusters); + mNMaxMergedTrackClusters = mRec->MemoryScalers()->NTPCMergedTrackHits(mNSectorHits); if (CAMath::Abs(Param().polynomialField.GetNominalBz()) < (gpu_common_constants::kZeroFieldCut * gpu_common_constants::kCLight)) { mNMaxTracks = mRec->MemoryScalers()->getValue(mNTotalSectorTracks, mNTotalSectorTracks); // 0 magnetic field } else { mNMaxTracks = mRec->MemoryScalers()->NTPCMergedTracks(mNTotalSectorTracks); } if (io.clustersNative) { - mNMaxClusters = io.clustersNative->nClustersTotal; + mNClusters = io.clustersNative->nClustersTotal; } else if (mRec->GetRecoSteps() & gpudatatypes::RecoStep::TPCSectorTracking) { - mNMaxClusters = 0; + mNClusters = 0; for (int32_t i = 0; i < NSECTORS; i++) { - mNMaxClusters += mRec->GetConstantMem().tpcTrackers[i].NHitsTotal(); + mNClusters += mRec->GetConstantMem().tpcTrackers[i].NHitsTotal(); } } else { - mNMaxClusters = mNClusters; + mNClusters = mNSectorHits; } - mNMaxLooperMatches = mNMaxClusters / 4; // We have that much scratch memory anyway + mNMaxLooperMatches = mNClusters / 4; // We have that much scratch memory anyway } int32_t GPUTPCGMMerger::CheckSectors() @@ -1656,7 +1661,7 @@ GPUd() void GPUTPCGMMerger::CollectMergedTracks(int32_t nBlocks, int32_t nThread const GPUTPCTracker& trk = GetConstantMem()->tpcTrackers[t->Sector()]; const GPUTPCHitId& ic = trk.TrackHits()[t->OrigTrack()->FirstHitID() + i]; uint32_t id = trk.Data().ClusterDataIndex(trk.Data().Row(ic.RowIndex()), ic.HitIndex()) + GetConstantMem()->ioPtrs.clustersNative->clusterOffset[t->Sector()][0]; - *c2 = trackCluster{.id = id, .row = (uint8_t)ic.RowIndex(), .sector = t->Sector(), .error = 0.f}; + *c2 = trackCluster{.id = id, .row = (uint8_t)ic.RowIndex(), .sector = t->Sector(), .error = 0.f, .weight = 0, .best = 0}; } nHits += nTrackHits; } @@ -1837,11 +1842,13 @@ GPUd() void GPUTPCGMMerger::PrepareForFit1(int32_t nBlocks, int32_t nThreads, in GPUTPCGMMergedTrack& trk = mMergedTracks[i]; if (trk.OK()) { for (uint32_t j = 0; j < trk.NClusters(); j++) { - uint32_t weight = attachAttached | attachGood; - if (CAMath::Abs(trk.GetParam().GetQPt() * Param().qptB5Scaler) <= Param().rec.tpc.rejectQPtB5 && !trk.MergedLooper() && trk.Leg() == 0) { - weight |= attachProtect; + if (!Param().rec.tpc.rebuildTrackInFit) { + uint32_t weight = attachAttached | attachGood; + if (CAMath::Abs(trk.GetParam().GetQPt() * Param().qptB5Scaler) <= Param().rec.tpc.rejectQPtB5 && !trk.MergedLooper() && trk.Leg() == 0) { + weight |= attachProtect; + } + CAMath::AtomicMax(&mClusterAttachment[mClusters[trk.FirstClusterRef() + j].num], weight); } - CAMath::AtomicMax(&mClusterAttachment[mClusters[trk.FirstClusterRef() + j].num], weight); CAMath::AtomicAdd(&mSharedCount[mClusters[trk.FirstClusterRef() + j].num], 1u); } if (!trk.CCE() && !trk.MergedLooper()) { @@ -1872,8 +1879,8 @@ GPUd() void GPUTPCGMMerger::PrepareForFit2(int32_t nBlocks, int32_t nThreads, in mClusters[i].state |= GPUTPCGMMergedTrackHit::flagShared; } } - if (mClusterStateExt) { - for (uint32_t i = iBlock * nThreads + iThread; i < mNMaxClusters; i += nBlocks * nThreads) { + if (!Param().rec.tpc.rebuildTrackInFit && mClusterStateExt) { + for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nBlocks * nThreads) { uint8_t state = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[i].getFlags(); if (mSharedCount[i] > 1) { state |= GPUTPCGMMergedTrackHit::flagShared; @@ -1924,7 +1931,7 @@ GPUd() void GPUTPCGMMerger::Finalize1(int32_t nBlocks, int32_t nThreads, int32_t GPUd() void GPUTPCGMMerger::Finalize2(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) { - for (uint32_t i = iBlock * nThreads + iThread; i < mNMaxClusters; i += nThreads * nBlocks) { + for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nThreads * nBlocks) { if (mClusterAttachment[i] != 0) { mClusterAttachment[i] = (mClusterAttachment[i] & attachFlagMask) | mTrackSort[mClusterAttachment[i] & attachTrackMask]; } @@ -2119,3 +2126,168 @@ GPUd() void GPUTPCGMMerger::MergeLoopersMain(int32_t nBlocks, int32_t nThreads, } #endif*/ } + +GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) +{ + for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nBlocks * nThreads) { + mSharedCount2[i] = 0; + } + if (iThread == 0 && iBlock == 0) { + mMemory->nMergedTrackClusters = 0; + } + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + const auto& trk = mMergedTracks[i]; + if (!trk.OK()) { + continue; + } + mTrackRebuildHelper[i].reverse = mClusters[trk.FirstClusterRef()].row < mClusters[trk.FirstClusterRef() + trk.NClusters() - 1].row; + int lastRow = -1; + for (uint32_t j = 0; j < trk.NClusters(); j++) { + const auto& cl = mClusters[trk.FirstClusterRef() + j]; + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (cl.row != lastRow && candidates[0].id == 0 && (!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0)) { + candidates[0].id = cl.num + 2; + candidates[0].best = 128; + candidates[0].weight = cl.state; + candidates[0].sector = cl.sector; + lastRow = cl.row; + } + } + } +} + +GPUd() void GPUTPCGMMerger::ComputeHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration) +{ + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + const auto& trk = mMergedTracks[i]; + if (!trk.OK() || trk.GetParam().GetNDF() <= 0) { + continue; + } + for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidates[0].best == 0) { + if (candidates[iteration].id >= 2) { + float weightf = trk.NClusters() * (trk.NClusters() * 2 - 5) * 128 / trk.GetParam().GetChi2() / (1.f + candidates[iteration].error); + int32_t weight = weightf < 0.f ? 0 : (weightf > 2e9f ? 2e9f : (int32_t)weightf); + candidates[iteration].weight = weight; + CAMath::AtomicMax(&mHitWeights[candidates[iteration].id - 2], weight); + CADEBUG(printf("REBUILD: iTrk %d Iteration %d Row %d Cluster %d Weight %d\n", i, iteration, j, candidates[iteration].id - 2, weight)); + } else { + candidates[0].best = -1; + } + } + } + } +} + +GPUd() void GPUTPCGMMerger::ResolveHitWeights1(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration) +{ + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + auto& trk = mMergedTracks[i]; + if (!trk.OK() || trk.GetParam().GetNDF() <= 0) { + continue; + } + for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidates[0].best == 0) { + if (candidates[iteration].weight == mHitWeights[candidates[iteration].id - 2]) { + candidates[0].best = iteration + 1; + } + } + } + } +} + +GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) +{ + int32_t nIter = Param().rec.tpc.rebuildTrackInFitClusterCandidates; + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + auto& trk = mMergedTracks[i]; + if (!trk.OK()) { + trk.SetNClusters(0); + continue; + } + uint32_t nHits = 0; + uint8_t sharedRowCandidates[GPUTPCGeometry::NROWS]; + uint8_t nSharedRowCandidates = 0; + for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (trk.GetParam().GetNDF() > 0 && candidates[0].best == 0 && candidates[nIter - 1].weight == mHitWeights[candidates[nIter - 1].id - 2]) { + candidates[0].best = nIter; + } + if (candidates[0].best > 0) { + nHits++; + } else if (candidates[0].id >= 2) { + sharedRowCandidates[nSharedRowCandidates++] = j; + } + } + + if (nSharedRowCandidates) { + const uint32_t nShared = CAMath::Min(nSharedRowCandidates, CAMath::Float2UIntRn(nHits * Param().rec.tpc.rebuildTrackMaxSharedFraction)); + if (nShared) { + const uint32_t n = Param().rec.tpc.rebuildTrackInFitClusterCandidates; + const auto* candidates = &mClusterCandidates[i * GPUTPCGeometry::NROWS * n]; + GPUCommonAlgorithm::sort(sharedRowCandidates, sharedRowCandidates + nSharedRowCandidates, [n, candidates](const uint8_t& a, const uint8_t& b) { + const auto& ca = candidates[a * n]; + const auto& cb = candidates[b * n]; + return GPUCA_DETERMINISTIC_CODE(ca.error == cb.error ? ca.id < cb.id : ca.error < cb.error, ca.error < cb.error); + }); + for (uint32_t j = 0; j < nShared; j++) { + auto* cj = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + sharedRowCandidates[j]) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + CADEBUG(printf("REBUILD: iTrk %d Recovering shared %d / %d: row %d %d, error %f\n", i, j, nShared, (int)sharedRowCandidates[j], cj[0].id - 2, cj[0].error)); + cj[0].best = 1; + mSharedCount2[cj[0].id - 2] = 1; // TODO: Should we also recover iter0 clusters? + } + nHits += nShared; + } + } + + const uint32_t iOut = CAMath::AtomicAdd(&mMemory->nMergedTrackClusters, nHits); + if (iOut + nHits > mNMaxMergedTrackClusters) { + raiseError(GPUErrors::ERROR_MERGER_REBUILD_HIT_OVERFLOW, iOut, mNMaxMergedTrackClusters); + CAMath::AtomicExch(&mMemory->nMergedTrackClusters, mNMaxMergedTrackClusters); + trk.SetNClusters(0); + trk.SetOK(false); + continue; + } + + trk.SetNClusters(nHits); + trk.SetFirstClusterRef(iOut); + uint32_t written = 0; + const bool reverse = mTrackRebuildHelper[i].reverse; + for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { + const auto& candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidates[0].best > 0) { + auto& outCl = mClusters[iOut + (reverse ? (written) : (nHits - 1 - written))]; + if (candidates[0].best == 128) { + outCl = {.num = candidates[0].id - 2, .sector = candidates[0].sector, .row = (uint8_t)j, .state = (uint8_t)candidates[0].weight}; + written++; + continue; + } + const auto& best = candidates[candidates[0].best - 1]; + const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[best.id - 2]; + outCl = {.num = best.id - 2, .sector = best.sector, .row = (uint8_t)j, .state = (uint8_t)(cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags)}; + written++; + CADEBUG(printf("REBUILD: iTrk %d Assigned Cluster Row %d Hit %d\n", i, j, best.id - 2)); + } + } + } +} + +GPUd() void GPUTPCGMMerger::ResolveHitWeightsShared(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) +{ + for (uint32_t i = iBlock * nThreads + iThread; i < mMemory->nMergedTrackClusters; i += nBlocks * nThreads) { + if (mSharedCount2[mClusters[i].num]) { + mClusters[i].state |= GPUTPCGMMergedTrackHit::flagShared; // TODO: What is the best criterion to set the shared flag? Perhaps already when iter0 best hit would be shared? + } + } + if (mClusterStateExt) { + for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nBlocks * nThreads) { + uint8_t state = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[i].getFlags(); + if (mSharedCount2[i]) { + state |= GPUTPCGMMergedTrackHit::flagShared; + } + mClusterStateExt[i] = state; + } + } +} diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 6be66bcc9447e..7346ab5829fad 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -83,6 +83,12 @@ class GPUTPCGMMerger : public GPUProcessor uint8_t row; uint8_t sector; float error; + int32_t weight; + int32_t best; + }; + + struct trackRebuildHelper { + bool reverse; }; struct tmpSort { @@ -119,14 +125,15 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() GPUdEdxInfo* MergedTracksdEdx() { return mMergedTracksdEdx; } GPUhdi() const GPUdEdxInfo* MergedTracksdEdxAlt() const { return mMergedTracksdEdxAlt; } GPUhdi() GPUdEdxInfo* MergedTracksdEdxAlt() { return mMergedTracksdEdxAlt; } + GPUhdi() uint32_t NSectorHits() const { return mNSectorHits; } GPUhdi() uint32_t NClusters() const { return mNClusters; } - GPUhdi() uint32_t NMaxClusters() const { return mNMaxClusters; } GPUhdi() uint32_t NMaxTracks() const { return mNMaxTracks; } GPUhdi() uint32_t NMaxMergedTrackClusters() const { return mNMaxMergedTrackClusters; } GPUhdi() uint32_t NMergedTrackClusters() const { return mMemory->nMergedTrackClusters; } GPUhdi() const GPUTPCGMMergedTrackHit* Clusters() const { return mClusters; } - GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return (mClusters); } - GPUhdi() trackCluster* ClusterCandidates() { return (mClusterCandidates); } + GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return mClusters; } + GPUhdi() trackCluster* ClusterCandidates() { return mClusterCandidates; } + GPUhdi() int32_t* HitWeights() { return mHitWeights; } GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } GPUhdi() uint32_t* TrackOrderProcess() const { return mTrackOrderProcess; } @@ -139,6 +146,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() tmpSort* TrackSortO2() { return mTrackSortO2; } GPUhdi() internal::MergeLooperParam* LooperCandidates() { return mLooperCandidates; } GPUhdi() GPUAtomic(uint32_t) * SharedCount() { return mSharedCount; } + GPUhdi() uint8_t* SharedCount2() { return mSharedCount2; } GPUhdi() gputpcgmmergertypes::GPUTPCGMBorderRange* BorderRange(int32_t i) { return mBorderRange[i]; } GPUhdi() const gputpcgmmergertypes::GPUTPCGMBorderRange* BorderRange(int32_t i) const { return mBorderRange[i]; } GPUhdi() GPUTPCGMBorderTrack* BorderTracks(int32_t i) { return mBorder[i]; } @@ -196,6 +204,11 @@ class GPUTPCGMMerger : public GPUProcessor GPUd() void MergeLoopersInit(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void MergeLoopersSort(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void MergeLoopersMain(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() void PrepareHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() void ComputeHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration); + GPUd() void ResolveHitWeights1(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration); + GPUd() void ResolveHitWeights2(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() void ResolveHitWeightsShared(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); #ifndef GPUCA_GPUCODE void DumpSectorTracks(std::ostream& out) const; @@ -261,7 +274,7 @@ class GPUTPCGMMerger : public GPUProcessor uint32_t mNMaxTracks = 0; // maximum number of output tracks uint32_t mNMaxSingleSectorTracks = 0; // max N tracks in one sector uint32_t mNMaxMergedTrackClusters = 0; // max number of clusters in output tracks (double-counting shared clusters) - uint32_t mNMaxClusters = 0; // max total unique clusters (in event) + uint32_t mNClusters = 0; // max total unique clusters (in event) uint32_t mNMaxLooperMatches = 0; // Maximum number of candidate pairs for looper matching uint16_t mMemoryResMemory = (uint16_t)-1; @@ -272,9 +285,12 @@ class GPUTPCGMMerger : public GPUProcessor uint16_t mMemoryResOutputO2MC = (uint16_t)-1; uint16_t mMemoryResOutputO2Scratch = (uint16_t)-1; - int32_t mNClusters = 0; // Total number of incoming clusters (from sector tracks) + int32_t mNSectorHits = 0; // Total number of incoming clusters (from sector tracks) GPUTPCGMMergedTrack* mMergedTracks = nullptr; //* array of output merged tracks trackCluster* mClusterCandidates = nullptr; + trackRebuildHelper* mTrackRebuildHelper = nullptr; + int32_t* mHitWeights = nullptr; + GPUdEdxInfo* mMergedTracksdEdx = nullptr; //* dEdx information GPUdEdxInfo* mMergedTracksdEdxAlt = nullptr; //* dEdx alternative information GPUTPCGMSectorTrack* mSectorTrackInfos = nullptr; //* additional information for sector tracks @@ -295,6 +311,7 @@ class GPUTPCGMMerger : public GPUProcessor uint32_t* mTrackSort = nullptr; tmpSort* mTrackSortO2 = nullptr; GPUAtomic(uint32_t) * mSharedCount = nullptr; // Must be uint32_t unfortunately for atomic support + uint8_t* mSharedCount2 = nullptr; GPUTPCGMBorderTrack* mBorderMemory = nullptr; // memory for border tracks GPUTPCGMBorderTrack* mBorder[2 * NSECTORS]; gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRangeMemory = nullptr; // memory for border tracks diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx index 9ee2bcf4eb6b5..0f1c801a0a6f8 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx @@ -192,7 +192,7 @@ void GPUTPCGMMerger::DumpFitPrepare(std::ostream& out) const out << " Track Clusters"; DumpTrackClusters(out); uint32_t j = 0; - for (uint32_t i = 0; i < mNMaxClusters; i++) { + for (uint32_t i = 0; i < mNClusters; i++) { if ((mClusterAttachment[i] & attachFlagMask) != 0) { if (j++ % 10 == 0) { out << "\n Cluster attachment "; @@ -241,7 +241,7 @@ void GPUTPCGMMerger::DumpFinal(std::ostream& out) const out << "Track Clusters\n"; DumpTrackClusters(out, true); uint32_t j = 0; - for (uint32_t i = 0; i < mNMaxClusters; i++) { + for (uint32_t i = 0; i < mNClusters; i++) { if ((mClusterAttachment[i] & attachFlagMask) != 0) { if (++j % 10 == 0) { out << " Cluster attachment "; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 22d8b159e9b74..d3ea60b4dc099 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -214,3 +214,34 @@ GPUdii() void GPUTPCGMMergerMergeLoopers::Thread<2>(int32_t nBlocks, int32_t nTh { merger.MergeLoopersMain(nBlocks, nThreads, iBlock, iThread); } + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.PrepareHitWeights(nBlocks, nThreads, iBlock, iThread); +} + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.ComputeHitWeights(nBlocks, nThreads, iBlock, iThread, iteration); +} + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.ResolveHitWeights1(nBlocks, nThreads, iBlock, iThread, iteration); +} + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.ResolveHitWeights2(nBlocks, nThreads, iBlock, iThread); +} + + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.ResolveHitWeightsShared(nBlocks, nThreads, iBlock, iThread); +} diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h index 0e3e4990ff04b..669c241c236d2 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h @@ -180,6 +180,19 @@ class GPUTPCGMMergerMergeLoopers : public GPUTPCGMMergerGeneral GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger); }; +class GPUTPCGMMergerHitWeights : public GPUTPCGMMergerGeneral +{ + public: + enum K { defaultKernel = 0, + prepare = 0, + compute = 1, + resolve1 = 2, + resolve2 = 3, + resolveShared = 4 }; + template + GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger, int32_t iteration); +}; + } // namespace o2::gpu #endif diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 0ee2ac74cb561..6650149b73792 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -136,26 +136,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ // CADEBUG(if ((uint32_t)merger->GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger->GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); // clang-format on - if (rebuilt && merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + clusters[ihit].row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id > 0) { - while (true) { - if (!(ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector)) { - break; - } - } - const auto& interHit = merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + clusters[ihit].row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; - if (interHit.id == 1) { - nMissed++; - nMissed2++; - continue; - } - const ClusterNative& GPUrestrict() cl = merger -> GetConstantMem()->ioPtrs.clustersNative->clustersLinear[interHit.id - 2]; - merger->GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); - if (interHit.id - 2 == clusters[ihit].num) { - clusterState = clusters[ihit].state; - } else { - clusterState = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; - } - } else if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { + if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { nMissed++; nMissed2++; continue; @@ -227,7 +208,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ GPUglobalref() const cahit2* hits = tracker.HitData(row); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); if (row.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - const float zOffset = merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin); + const float zOffset = param.par.continuousTracking ? merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = row.Grid().YMin(); const float stepY = row.HstepY(); const float z0 = row.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway @@ -303,13 +284,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; } - merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = cluster.row, .sector = cluster.sector, .error = err}; + merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = cluster.row, .sector = cluster.sector, .error = err, .weight = 0, .best = 0}; nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); } } } } } + CADEBUG(const auto* dbgCand = &merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("iTrk %d Row %d Hit %d Candidate %d hit %d err %f\n", iTrk, (int)cluster.row, cluster.num, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); } if (nCandidates == 0) { merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; @@ -1042,9 +1024,13 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger->MergedTracks()[iTrk].Looper()); }); - track.SetOK(ok); - track.Param() = t; - track.Alpha() = Alpha; + track.SetOK(ok); // TODO: Should we recover tracks who failed the fit in iWay0/1 for the rebuild? + if (t.GetNDF() <= 0 && !rebuilt && merger->Param().rec.tpc.rebuildTrackInFit) { // TODO: Better handling of NDF<0 tracks, how do we want to do cluster rejection? + track.Param().NDF() = 0; + } else { + track.Param() = t; + track.Alpha() = Alpha; + } if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) { track.SetNClustersFitted(nTrackHits); } diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h b/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h index 7c9047202417b..aaf1bff882522 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h @@ -162,9 +162,12 @@ class GPUTPCTracker : public GPUProcessor GPUdi() static int32_t CalculateHitWeight(int32_t NHits, float chi2) { float weight = NHits * (NHits * 2 - 5) * 128 / chi2; // TODO: Add QPt to this formula - if (weight < 0.f || weight > 2e9f) { + if (weight < 0.f) { return 0; } + if (weight > 2e9f) { + return 2e9f; + } return ((int32_t)weight); } GPUdi() float GetChiSeedFactor() const { return Param().rec.tpc.hitPickUpFactor * Param().rec.tpc.hitPickUpFactor * 3.5f * 3.5f; } diff --git a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx index 52df66d9e69af..85a2b718c0595 100644 --- a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx +++ b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx @@ -962,7 +962,7 @@ int32_t main(int argc, char** argv) nEventsProcessed++; if (configStandalone.timeFrameTime) { - double nClusters = chainTracking->GetProcessors()->tpcMerger.NMaxClusters(); + double nClusters = chainTracking->GetProcessors()->tpcMerger.NClusters(); if (nClusters > 0) { const int32_t nOrbits = 32; const double colRate = 50000; diff --git a/GPU/GPUTracking/kernels.cmake b/GPU/GPUTracking/kernels.cmake index 93b5dfdf6158c..c2bea51f1512c 100644 --- a/GPU/GPUTracking/kernels.cmake +++ b/GPU/GPUTracking/kernels.cmake @@ -84,6 +84,11 @@ o2_gpu_add_kernel("GPUTPCGMMergerFinalize, step2" "GPUTP o2_gpu_add_kernel("GPUTPCGMMergerMergeLoopers, step0" "GPUTPCGMMergerGPU TPCMERGER" LB) o2_gpu_add_kernel("GPUTPCGMMergerMergeLoopers, step1" "GPUTPCGMMergerGPU TPCMERGER" LB) o2_gpu_add_kernel("GPUTPCGMMergerMergeLoopers, step2" "GPUTPCGMMergerGPU TPCMERGER" LB) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, prepare" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, compute" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, resolve1" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, resolve2" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, resolveShared" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) o2_gpu_add_kernel("GPUTPCGMO2Output, prepare" "= TPCMERGER" LB) o2_gpu_add_kernel("GPUTPCGMO2Output, sort" "= TPCMERGER") o2_gpu_add_kernel("GPUTPCGMO2Output, output" "= TPCMERGER" LB) diff --git a/GPU/GPUTracking/qa/GPUQA.cxx b/GPU/GPUTracking/qa/GPUQA.cxx index 5bbb0e2546a13..9f24f3745c45c 100644 --- a/GPU/GPUTracking/qa/GPUQA.cxx +++ b/GPU/GPUTracking/qa/GPUQA.cxx @@ -1830,7 +1830,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } } - uint32_t nCl = clNative ? clNative->nClustersTotal : mTracking->GetProcessors()->tpcMerger.NMaxClusters(); + uint32_t nCl = clNative ? clNative->nClustersTotal : mTracking->GetProcessors()->tpcMerger.NClusters(); mClusterCounts.nTotal += nCl; if (mQATasks & (taskClusterCounts | taskClusterRejection)) { for (uint32_t iSector = 0; iSector < GPUTPCGeometry::NSECTORS; iSector++) { From b1edead78804e9ceecfa8e11beef01742f01a46f Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 19 Feb 2026 20:39:07 +0100 Subject: [PATCH 06/25] GPU: Additional debug dumps --- .../Global/GPUChainTrackingMerger.cxx | 2 + GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 2 + GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx | 37 +++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 60dd8f591e0f5..de56c4d21c079 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -243,7 +243,9 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } } runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); + DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpInterpolatedHits, *mDebugFile); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); + DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRebuiltTracks, *mDebugFile); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); } runKernel(GetGridAuto(0)); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 7346ab5829fad..22dd41a6e7a5b 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -222,6 +222,8 @@ class GPUTPCGMMerger : public GPUProcessor void DumpRefit(std::ostream& out) const; void DumpFinal(std::ostream& out) const; void DumpLoopers(std::ostream& out) const; + void DumpInterpolatedHits(std::ostream& out) const; + void DumpRebuiltTracks(std::ostream& out) const; void DumpTrackParam(std::ostream& out) const; void DumpTrackClusters(std::ostream& out, bool non0StateOnly = false, bool noNDF0 = false) const; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx index 0f1c801a0a6f8..12006527b7d87 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx @@ -150,6 +150,15 @@ void GPUTPCGMMerger::DumpTrackParam(std::ostream& out) const out << std::setprecision(ss); } +void GPUTPCGMMerger::DumpRebuiltTracks(std::ostream& out) const +{ + out << "\nTPC Merger Rebuilt Tracks\n"; + out << " Cluster Attachment\n"; + DumpTrackClusters(out, false, true); + out << " Track Params\n"; + DumpTrackParam(out); +} + void GPUTPCGMMerger::DumpMergeCE(std::ostream& out) const { DumpTrackLinks(out, true, " for CE merging"); @@ -255,6 +264,34 @@ void GPUTPCGMMerger::DumpFinal(std::ostream& out) const out << "\n"; } +void GPUTPCGMMerger::DumpInterpolatedHits(std::ostream& out) const +{ + out << "\nTPC Merger Interpolated Hits\n"; + for (uint32_t i = 0; i < mMemory->nMergedTracks; i++) { + const auto& trk = mMergedTracks[i]; + if (trk.OK() && trk.GetParam().GetNDF() >= 0 && trk.NClusters()) { + out << "Track " << i << ":"; + for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidates[0].id) { + out << " Row " << j << ": "; + if (candidates[0].best) { + out << " Best " << candidates[0].best << " - "; + } + for (uint32_t k = 0; k < Param().rec.tpc.rebuildTrackInFitClusterCandidates; k++) { + if (candidates[k].id) { + out << k << ": id " << candidates[k].id << " err " << candidates[k].error << " weight " << candidates[k].weight << " - "; + } + } + out << "; "; + } + } + out << "\n"; + } + } + out << "\n"; +} + template inline void GPUTPCGMMerger::MergedTrackStreamerInternal(const GPUTPCGMBorderTrack& b1, const GPUTPCGMBorderTrack& b2, const char* name, int32_t sector1, int32_t sector2, uint8_t mergeMode, float weight, float frac) const { From ade7354bd2f173a55acaf4bdaf5582e176fb5dd6 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 6 Oct 2025 22:09:55 +0200 Subject: [PATCH 07/25] GPU TPC: Split TPC Merger Fit() into multiple functions --- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 4 +- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 5 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 652 +++++++++--------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 46 +- 4 files changed, 377 insertions(+), 330 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 49d041408cb00..8083702bc3e98 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -571,7 +571,7 @@ GPUd() int32_t GPUTPCGMMerger::RefitSectorTrack(GPUTPCGMSectorTrack& sectorTrack trk.QPt() = inTrack->Param().GetQPt(); trk.TOffset() = Param().par.continuousTracking ? GetConstantMem()->calibObjects.fastTransform->convZOffsetToVertexTime(sector, inTrack->Param().GetZOffset(), Param().continuousMaxTimeBin) : 0; const auto tmp = sectorTrack.ClusterTN() > sectorTrack.ClusterT0() ? std::array{sectorTrack.ClusterTN(), sectorTrack.ClusterT0()} : std::array{sectorTrack.ClusterT0(), sectorTrack.ClusterTN()}; - trk.ShiftZ(this, sector, tmp[0], tmp[1], inTrack->Param().GetX()); // We do not store the inner / outer cluster X, so we just use the track X instead + trk.ShiftZ(*this, sector, tmp[0], tmp[1], inTrack->Param().GetX()); // We do not store the inner / outer cluster X, so we just use the track X instead sectorTrack.SetX2(0.f); for (int32_t way = 0; way < 2; way++) { if (way) { @@ -1857,7 +1857,7 @@ GPUd() void GPUTPCGMMerger::PrepareForFit1(int32_t nBlocks, int32_t nThreads, in const auto& GPUrestrict() cls = GetConstantMem()->ioPtrs.clustersNative->clustersLinear; float z0 = cls[cl0.num].getTime(), zn = cls[cln.num].getTime(); const auto tmp = zn > z0 ? std::array{zn, z0, GPUTPCGeometry::Row2X(cln.row)} : std::array{z0, zn, GPUTPCGeometry::Row2X(cl0.row)}; - trk.Param().ShiftZ(this, cl0.sector, tmp[0], tmp[1], tmp[2]); + trk.Param().ShiftZ(*this, cl0.sector, tmp[0], tmp[1], tmp[2]); updTrk = &trk; while (updTrk->PrevSegment() >= 0) { auto next = &mMergedTracks[updTrk->PrevSegment()]; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index d3ea60b4dc099..083cdf76699a0 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -23,7 +23,7 @@ GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThread { GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, merger.NMergedTracks(), { const int32_t i = mode ? merger.TrackOrderProcess()[ii] : ii; - GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger, rebuilt); + GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, merger, rebuilt); }); } @@ -31,7 +31,7 @@ template <> GPUdii() void GPUTPCGMMergerFollowLoopers::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger) { GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), uint32_t, i, merger.Memory()->nLoopData, { - GPUTPCGMTrackParam::PropagateLooper(&merger, i); + GPUTPCGMTrackParam::PropagateLooper(merger, i); }); } @@ -239,7 +239,6 @@ GPUdii() void GPUTPCGMMergerHitWeights::Thread GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 6650149b73792..5dddb2b063f57 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -47,13 +47,13 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { - static constexpr float kDeg2Rad = M_PI / 180.f; static constexpr float maxSinPhi = constants::MAX_SIN_PHI; CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); - const GPUParam& GPUrestrict() param = merger->Param(); + const GPUParam& GPUrestrict() param = merger.Param(); + GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); GPUdEdx dEdx, dEdxAlt; GPUTPCGMPropagator prop; @@ -86,13 +86,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const bool refit = (nWays == 1 || iWay >= 1); const bool finalOutInFit = iWay + 2 >= nWays; const bool finalFit = iWay == nWays - 1; - const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); ResetCovariance(); prop.SetSeedingErrors(!(refit)); prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed prop.SetPropagateBzOnly(param.rec.fitPropagateBzOnly == -1 ? !finalFit : param.rec.fitPropagateBzOnly); - prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger->GetConstantMem()->calibObjects.matLUT : nullptr); + prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger.GetConstantMem()->calibObjects.matLUT : nullptr); prop.SetTrack(this, iWay && !rebuilt ? prop.GetAlpha() : Alpha); ConstrainSinPhi(iWay == 0 ? 0.95f : constants::MAX_SIN_PHI_LOW); CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / kSectAngle) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); @@ -128,13 +127,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; { - const ClusterNative& GPUrestrict() cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; - merger->GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); } // clang-format off CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); - // CADEBUG(if ((uint32_t)merger->GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) - // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger->GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); + // CADEBUG(if ((uint32_t)merger.GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) + // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger.GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); // clang-format on if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { nMissed++; @@ -169,253 +168,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } } - int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); - // clang-format off - CADEBUG(if (!CheckCov()){printf("INVALID COV AFTER PROPAGATE!!!\n");}); - // clang-format on - if (retValProp == -2) // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha - { - CADEBUG(printf("REROTATE\n")); - if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) == 0) { - retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); - } - } - if (retValProp == 0) { - lastRow = cluster.row; - lastSector = cluster.sector; - } - // clang-format off - CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); - // clang-format on - - if (crossCE) { - if (param.rec.tpc.addErrorsCECrossing) { - if (param.rec.tpc.addErrorsCECrossing >= 2) { - AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); - } else { - AddCovDiagErrors(param.rec.tpc.errorsCECrossing); - } - } else if (mC[2] < 0.5f) { - mC[2] = 0.5f; - } - } - auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; - float uncorrectedY = -1e6f; - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { - const GPUTPCTracker& GPUrestrict() tracker = *(merger -> GetConstantMem()->tpcTrackers + cluster.sector); - const GPUTPCRow& GPUrestrict() row = tracker.Row(cluster.row); - GPUglobalref() const cahit2* hits = tracker.HitData(row); - GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); - if (row.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - const float zOffset = param.par.continuousTracking ? merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin) : 0; - const float y0 = row.Grid().YMin(); - const float stepY = row.HstepY(); - const float z0 = row.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway - const float stepZ = row.HstepZ(); - int32_t bin, ny, nz; - - float err2Y, err2Z; - param.GetClusterErrors2(cluster.sector, cluster.row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge - - const float Iz0 = inter.posY - mP[0]; - const float Iz1 = inter.posZ + deltaZ - mP[1]; - const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); - const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); - const float Ik00 = mC[0] * Iw0; - const float Ik11 = mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0; - const float ImP1 = mP[1] + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0]; - const float ImC2 = mC[2] - Ik11 * mC[2]; - - float uncorrectedZ; - merger->GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, ImP0, ImP1, uncorrectedY, uncorrectedZ); - - int32_t nCandidates = 0; - if (CAMath::Abs(uncorrectedY) <= row.getTPCMaxY()) { - const float kFactor = tracker.GetChiSeedFactor(); - const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? - const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(mC[2]))); - const float tubeY = CAMath::Sqrt(sy2); - const float tubeZ = CAMath::Sqrt(sz2); - row.Grid().GetBinArea(uncorrectedY, uncorrectedZ + zOffset, tubeY, tubeZ, bin, ny, nz); - - const int32_t nBinsY = row.Grid().Ny(); - const int32_t idOffset = tracker.Data().ClusterIdOffset(); - const int32_t* ids = &(tracker.Data().ClusterDataIndex()[row.HitNumberOffset()]); - for (int32_t k = 0; k <= nz; k++) { - const int32_t mybin = bin + k * nBinsY; - const uint32_t hitFst = firsthit[mybin]; - const uint32_t hitLst = firsthit[mybin + ny + 1]; - for (uint32_t ih = hitFst; ih < hitLst; ih++) { - const cahit2 hh = hits[ih]; - const float y = y0 + hh.x * stepY; - const float z = z0 + hh.y * stepZ; - const float dy = y - uncorrectedY; - const float dz = z - uncorrectedZ; - - if (dy * dy < sy2 && dz * dz < sz2) { - float err2YA, err2ZA; - const ClusterNative& GPUrestrict() cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[idOffset + ids[ih]]; - const auto clflags = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; - const float time = cl.getTime(); - const float invSqrtCharge = CAMath::InvSqrt(cl.qMax); - const float invCharge = 1.f / cl.qMax; - float invAvgCharge = (sumInvSqrtCharge + invSqrtCharge) / (nAvgCharge + 1); - invAvgCharge *= invAvgCharge; - - prop.GetErr2(err2YA, err2ZA, merger->Param(), mP[1], cluster.row, clflags, cluster.sector, time, invAvgCharge, invCharge); - const float Jw0 = 1.f / (ImC0 + err2YA); - const float Jw2 = 1.f / (ImC2 + err2ZA); - const float chi2Y = Jw0 * dy * dy; - const float chi2Z = Jw2 * dz * dz; - bool ok = !prop.RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clflags); - float err = dy * dy + dz * dz; - if (ok) { - int32_t insert = nCandidates; - for (int32_t c = 0; c < nCandidates; c++) { - if (err < merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c].error) { - insert = c; - break; - } - } - if (insert < param.rec.tpc.rebuildTrackInFitClusterCandidates) { - for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { - merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; - } - merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = cluster.row, .sector = cluster.sector, .error = err, .weight = 0, .best = 0}; - nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); - } - } - } - } - } - CADEBUG(const auto* dbgCand = &merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("iTrk %d Row %d Hit %d Candidate %d hit %d err %f\n", iTrk, (int)cluster.row, cluster.num, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); - } - if (nCandidates == 0) { - merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; - } - } - } else if (allowChangeClusters) { - uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); - } - const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; - if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { + int32_t retValHit = FitHit(merger, iTrk, track, N, NTolerated, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, lastRow, lastSector, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, ihitStart, allowChangeClusters, refit, finalFit, nMissed, nMissed2, covYYUpd, resetT0); + if (retValHit == 1) { break; - } - if (retValProp || sinPhiErr) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - nMissed2++; - NTolerated++; - CADEBUG(printf(", %d --- break\n", (int32_t)sinPhiErr)); + } else if (retValHit == -1) { continue; } - CADEBUG(printf("\n")); - - int32_t retValUpd = 0, retValInt = 0; - float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); - if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { - retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; - } else { - int8_t rejectChi2 = 0; - if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { - if (iWay == nWays - 3) { - rejectChi2 = GPUTPCGMPropagator::rejectInterFill; - } else if (iWay == nWays - 2) { - rejectChi2 = GPUTPCGMPropagator::rejectInterReject; - } else if (iWay == nWays - 1) { - rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; - } - } - - float err2Y, err2Z; - const float time = merger->GetConstantMem()->ioPtrs.clustersNative ? merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].getTime() : -1.f; - const float invSqrtCharge = merger->GetConstantMem()->ioPtrs.clustersNative ? CAMath::InvSqrt(merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; - const float invCharge = merger->GetConstantMem()->ioPtrs.clustersNative ? (1.f / merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; - float invAvgCharge = (sumInvSqrtCharge += invSqrtCharge) / ++nAvgCharge; - invAvgCharge *= invAvgCharge; - prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); - - if (rejectChi2 >= GPUTPCGMPropagator::rejectInterFill) { - if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - rejectChi2 = GPUTPCGMPropagator::rejectDirect; - } else { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &inter, err2Y, err2Z, deltaZ); - } - } - - if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { // uncorrectedY > -1e6f implies allowChangeClusters - retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedEdge; - } else { - retValUpd = prop.Update(yy, zz, cluster.row, param, clusterState, rejectChi2, refit, err2Y, err2Z); - } - GPUCA_DEBUG_STREAMER_CHECK(if (o2::utils::DebugStreamer::checkStream(o2::utils::StreamFlags::streamUpdateTrack, iTrk)) { - merger->DebugStreamerUpdate(iTrk, ihit, xx, yy, zz, cluster, merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num], *this, prop, inter, rejectChi2, refit, retValUpd, sumInvSqrtCharge / nAvgCharge * sumInvSqrtCharge / nAvgCharge, yy, zz, clusterState, retValInt, err2Y, err2Z); - }); - } - // clang-format off - CADEBUG(if (!CheckCov()) GPUError("INVALID COV AFTER UPDATE!!!")); - CADEBUG(printf("\t%21sFit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd, retValInt)); - // clang-format on - - ConstrainSinPhi(); // TODO: Limit using ConstrainSinPhi everywhere! - if (!retValUpd && !retValInt) // track is updated - { - lastUpdateX = mX; - covYYUpd = mC[0]; - nMissed = nMissed2 = 0; - UnmarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - N++; - ihitStart = ihit; - float dy = mP[0] - prop.Model().Y(); - float dz = mP[1] - prop.Model().Z(); - if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && dy * dy + dz * dz > 1) { - CADEBUG(printf("Reinit linearization\n")); - prop.SetTrack(this, prop.GetAlpha()); - } - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { - if (param.dodEdxEnabled && finalFit) { // TODO: Costimize flag to remove, and option to remove double-clusters - bool acc = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMask) == 0, accAlt = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMaskAlt) == 0; - if (acc || accAlt) { - float qtot = 0, qmax = 0, pad = 0, relTime = 0; - const int32_t clusterCount = (ihit - ihitMergeFirst) * wayDirection + 1; - for (int32_t iTmp = ihitMergeFirst; iTmp != ihit + wayDirection; iTmp += wayDirection) { - const ClusterNative& cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num]; - qtot += cl.qTot; - qmax = CAMath::Max(qmax, cl.qMax); - pad += cl.getPad(); - relTime += cl.getTime(); - } - qtot /= clusterCount; // TODO: Weighted Average - pad /= clusterCount; - relTime /= clusterCount; - relTime = relTime - CAMath::Round(relTime); - if (acc) { - dEdx.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], merger->GetConstantMem()->calibObjects, zz, pad, relTime); - } - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { - if (accAlt) { - dEdxAlt.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], merger->GetConstantMem()->calibObjects, zz, pad, relTime); - } - } - } - } - } - } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track - if (retValInt || allowChangeClusters) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); - } else if (finalFit) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); - } - if (!retValInt) { - nMissed++; - nMissed2++; - } - } else { - break; // bad chi2 for the whole track, stop the fit - } } if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255 && lastSector != 255) { StoreLoopPropagation(merger, lastSector, lastRow, iTrk, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); @@ -449,9 +209,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } if (param.par.dodEdx && param.dodEdxEnabled) { - dEdx.computedEdx(merger->MergedTracksdEdx()[iTrk], param); + dEdx.computedEdx(merger.MergedTracksdEdx()[iTrk], param); if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { - dEdxAlt.computedEdx(merger->MergedTracksdEdxAlt()[iTrk], param); + dEdxAlt.computedEdx(merger.MergedTracksdEdxAlt()[iTrk], param); } } Alpha = prop.GetAlpha(); @@ -461,6 +221,277 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ return true; } +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, int32_t& N, int32_t& NTolerated, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, uint8_t& lastRow, uint8_t& lastSector, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, int32_t& ihitStart, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float& covYYUpd, int32_t& resetT0) +{ + const GPUParam& GPUrestrict() param = merger.Param(); + const int32_t nWays = param.rec.tpc.nWays; + const int32_t wayDirection = (iWay & 1) ? -1 : 1; + const auto& cluster = clusters[ihit]; + int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); + // clang-format off + CADEBUG(if (!CheckCov()){printf("INVALID COV AFTER PROPAGATE!!!\n");}); + // clang-format on + if (retValProp == -2) // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha + { + CADEBUG(printf("REROTATE\n")); + if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) == 0) { + retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); + } + } + if (retValProp == 0) { + lastRow = cluster.row; + lastSector = cluster.sector; + } + // clang-format off + CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); + // clang-format on + + if (crossCE) { + if (param.rec.tpc.addErrorsCECrossing) { + if (param.rec.tpc.addErrorsCECrossing >= 2) { + AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); + } else { + AddCovDiagErrors(param.rec.tpc.errorsCECrossing); + } + } else if (mC[2] < 0.5f) { + mC[2] = 0.5f; + } + } + + float uncorrectedY = -1e6f; + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + } else if (allowChangeClusters) { + uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); + } + + static constexpr float kDeg2Rad = M_PI / 180.f; + const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); + const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; + if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { + return 1; + } + if (retValProp || sinPhiErr) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); + nMissed2++; + NTolerated++; + CADEBUG(printf(", %d --- break\n", (int32_t)sinPhiErr)); + return -1; + } + CADEBUG(printf("\n")); + + int32_t retValUpd = 0, retValInt = 0; + float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); + if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { + retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; + } else { + int8_t rejectChi2 = 0; + if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { + if (iWay == nWays - 3) { + rejectChi2 = GPUTPCGMPropagator::rejectInterFill; + } else if (iWay == nWays - 2) { + rejectChi2 = GPUTPCGMPropagator::rejectInterReject; + } else if (iWay == nWays - 1) { + rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; + } + } + + float err2Y, err2Z; + const float time = merger.GetConstantMem()->ioPtrs.clustersNative ? merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].getTime() : -1.f; + const float invSqrtCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? CAMath::InvSqrt(merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; + const float invCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? (1.f / merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; + float invAvgCharge = (sumInvSqrtCharge += invSqrtCharge) / ++nAvgCharge; + invAvgCharge *= invAvgCharge; + + prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); + + if (rejectChi2 >= GPUTPCGMPropagator::rejectInterFill) { + if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + rejectChi2 = GPUTPCGMPropagator::rejectDirect; + } else { + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &inter, err2Y, err2Z, deltaZ); + } + } + + if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { // uncorrectedY > -1e6f implies allowChangeClusters + retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedEdge; + } else { + retValUpd = prop.Update(yy, zz, cluster.row, param, clusterState, rejectChi2, refit, err2Y, err2Z); + } + GPUCA_DEBUG_STREAMER_CHECK(if (o2::utils::DebugStreamer::checkStream(o2::utils::StreamFlags::streamUpdateTrack, iTrk)) { + merger.DebugStreamerUpdate(iTrk, ihit, xx, yy, zz, cluster, merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num], *this, prop, inter, rejectChi2, refit, retValUpd, sumInvSqrtCharge / nAvgCharge * sumInvSqrtCharge / nAvgCharge, yy, zz, clusterState, retValInt, err2Y, err2Z); + }); + } + // clang-format off + CADEBUG(if (!CheckCov()) GPUError("INVALID COV AFTER UPDATE!!!")); + CADEBUG(printf("\t%21sFit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd, retValInt)); + // clang-format on + + ConstrainSinPhi(); // TODO: Limit using ConstrainSinPhi everywhere! + if (!retValUpd && !retValInt) // track is updated + { + lastUpdateX = mX; + covYYUpd = mC[0]; + nMissed = nMissed2 = 0; + UnmarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); + N++; + ihitStart = ihit; + float dy = mP[0] - prop.Model().Y(); + float dz = mP[1] - prop.Model().Z(); + if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && dy * dy + dz * dz > 1) { + CADEBUG(printf("Reinit linearization\n")); + prop.SetTrack(this, prop.GetAlpha()); + } + FitdEdx(dEdx, dEdxAlt, param, finalFit, merger.GetConstantMem()->calibObjects, ihit, ihitMergeFirst, wayDirection, merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear, clusters, clusterState, zz); + } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track + if (retValInt || allowChangeClusters) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); + } else if (finalFit) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); + } + if (!retValInt) { + nMissed++; + nMissed2++; + } + } else { + return 1; // bad chi2 for the whole track, stop the fit + } + return 0; +} + +GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk) +{ + const GPUParam& GPUrestrict() param = merger.Param(); + const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); + const GPUTPCRow& GPUrestrict() rowData = tracker.Row(row); + GPUglobalref() const cahit2* hits = tracker.HitData(rowData); + GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(rowData); + float uncorrectedY = -1e6f, uncorrectedZ; + if (rowData.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; + const float y0 = rowData.Grid().YMin(); + const float stepY = rowData.HstepY(); + const float z0 = rowData.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway + const float stepZ = rowData.HstepZ(); + int32_t bin, ny, nz; + + float err2Y, err2Z; + param.GetClusterErrors2(sector, row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + + const float Iz0 = inter.posY - mP[0]; + const float Iz1 = inter.posZ + deltaZ - mP[1]; + const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); + const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); + const float Ik00 = mC[0] * Iw0; + const float Ik11 = mC[2] * Iw2; + const float ImP0 = mP[0] + Ik00 * Iz0; + const float ImP1 = mP[1] + Ik11 * Iz1; + const float ImC0 = mC[0] - Ik00 * mC[0]; + const float ImC2 = mC[2] - Ik11 * mC[2]; + + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, row, ImP0, ImP1, uncorrectedY, uncorrectedZ); + + int32_t nCandidates = 0; + if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { + const float kFactor = tracker.GetChiSeedFactor(); + const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? + const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(mC[2]))); + const float tubeY = CAMath::Sqrt(sy2); + const float tubeZ = CAMath::Sqrt(sz2); + rowData.Grid().GetBinArea(uncorrectedY, uncorrectedZ + zOffset, tubeY, tubeZ, bin, ny, nz); + + const int32_t nBinsY = rowData.Grid().Ny(); + const int32_t idOffset = tracker.Data().ClusterIdOffset(); + const int32_t* ids = &(tracker.Data().ClusterDataIndex()[rowData.HitNumberOffset()]); + for (int32_t k = 0; k <= nz; k++) { + const int32_t mybin = bin + k * nBinsY; + const uint32_t hitFst = firsthit[mybin]; + const uint32_t hitLst = firsthit[mybin + ny + 1]; + for (uint32_t ih = hitFst; ih < hitLst; ih++) { + const cahit2 hh = hits[ih]; + const float y = y0 + hh.x * stepY; + const float z = z0 + hh.y * stepZ; + const float dy = y - uncorrectedY; + const float dz = z - uncorrectedZ; + + if (dy * dy < sy2 && dz * dz < sz2) { + float err2YA, err2ZA; + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[idOffset + ids[ih]]; + const auto clflags = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; + const float time = cl.getTime(); + const float invSqrtCharge = CAMath::InvSqrt(cl.qMax); + const float invCharge = 1.f / cl.qMax; + float invAvgCharge = (sumInvSqrtCharge + invSqrtCharge) / (nAvgCharge + 1); + invAvgCharge *= invAvgCharge; + + prop.GetErr2(err2YA, err2ZA, param, mP[1], row, clflags, sector, time, invAvgCharge, invCharge); + const float Jw0 = 1.f / (ImC0 + err2YA); + const float Jw2 = 1.f / (ImC2 + err2ZA); + const float chi2Y = Jw0 * dy * dy; + const float chi2Z = Jw2 * dz * dz; + bool ok = !prop.RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clflags); + float err = dy * dy + dz * dz; + if (ok) { + int32_t insert = nCandidates; + for (int32_t c = 0; c < nCandidates; c++) { + if (err < merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c].error) { + insert = c; + break; + } + } + if (insert < param.rec.tpc.rebuildTrackInFitClusterCandidates) { + for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { + merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; + } + merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = row, .sector = sector, .error = err, .weight = 0, .best = 0}; + nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); + } + } + } + } + } + CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("iTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); + } + if (nCandidates == 0) { + merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; + } + } + return uncorrectedY; +} + +GPUdii() void GPUTPCGMTrackParam::FitdEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, const GPUParam& GPUrestrict() param, bool finalFit, const GPUCalibObjectsConst& GPUrestrict() calib, int ihit, int ihitMergeFirst, int wayDirection, const ClusterNative* GPUrestrict() clustersArray, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz) +{ + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { + if (param.dodEdxEnabled && finalFit) { // TODO: Costimize flag to remove, and option to remove double-clusters + bool acc = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMask) == 0, accAlt = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMaskAlt) == 0; + if (acc || accAlt) { + float qtot = 0, qmax = 0, pad = 0, relTime = 0; + const int32_t clusterCount = CAMath::Abs(ihit - ihitMergeFirst) + 1; + for (int32_t iTmp = ihitMergeFirst; iTmp != ihit + wayDirection; iTmp += wayDirection) { + const ClusterNative& cl = clustersArray[clusters[iTmp].num]; + qtot += cl.qTot; + qmax = CAMath::Max(qmax, cl.qMax); + pad += cl.getPad(); + relTime += cl.getTime(); + } + qtot /= clusterCount; // TODO: Weighted Average + pad /= clusterCount; + relTime /= clusterCount; + relTime = relTime - CAMath::Round(relTime); + const auto& cluster = clusters[ihit]; + if (acc) { + dEdx.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], calib, zz, pad, relTime); + } + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { + if (accAlt) { + dEdxAlt.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], calib, zz, pad, relTime); + } + } + } + } + } +} + GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& Alpha) { static constexpr float kDeg2Rad = M_PI / 180.f; @@ -518,11 +549,11 @@ GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, mChi2 = 0; } -GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger* GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, bool rejectChi2) +GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, bool rejectChi2) { if (ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector) { float maxDistY, maxDistZ; - prop.GetErr2(maxDistY, maxDistZ, merger->Param(), zz, clusters[ihit].row, 0, clusters[ihit].sector, -1.f, 0.f, 0.f); // TODO: Use correct time, avgCharge + prop.GetErr2(maxDistY, maxDistZ, merger.Param(), zz, clusters[ihit].row, 0, clusters[ihit].sector, -1.f, 0.f, 0.f); // TODO: Use correct time, avgCharge maxDistY = (maxDistY + mC[0]) * 20.f; maxDistZ = (maxDistZ + mC[2]) * 20.f; int32_t noReject = 0; // Cannot reject if simple estimation of y/z fails (extremely unlike case) @@ -537,10 +568,10 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t xx = yy = zz = 0.f; clusterState = 0; while (true) { - const ClusterNative& GPUrestrict() cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; float clamp = cl.qTot; float clx, cly, clz; - merger->GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), clx, cly, clz, mTOffset); + merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), clx, cly, clz, mTOffset); float dy = cly - projY; float dz = clz - projZ; if (noReject == 0 && (dy * dy > maxDistY || dz * dz > maxDistZ)) { @@ -572,25 +603,25 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t return 0; } -GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop) +GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop) { float Y, Z; float X = 0; - Merger->GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(sector, iRow, mP[0], mP[1], X); + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(sector, iRow, mP[0], mP[1], X); if (prop.GetPropagatedYZ(X, Y, Z)) { Y = mP[0]; Z = mP[1]; } - return AttachClusters(Merger, sector, iRow, iTrack, goodLeg, Y, Z); + return AttachClusters(merger, sector, iRow, iTrack, goodLeg, Y, Z); } -GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z) +GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z) { - const auto& param = Merger->Param(); + const GPUParam& GPUrestrict() param = merger.Param(); if (param.rec.tpc.disableRefitAttachment & 1) { return -1e6f; } - const GPUTPCTracker& GPUrestrict() tracker = *(Merger->GetConstantMem()->tpcTrackers + sector); + const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); const GPUTPCRow& GPUrestrict() row = tracker.Row(iRow); GPUglobalref() const cahit2* hits = tracker.HitData(row); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); @@ -598,7 +629,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric return -1e6f; } - const float zOffset = param.par.continuousTracking ? Merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; // TODO: do some validatiomns for the transform conv functions... + const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; // TODO: do some validatiomns for the transform conv functions... const float y0 = row.Grid().YMin(); const float stepY = row.HstepY(); const float z0 = row.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway @@ -606,7 +637,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric int32_t bin, ny, nz; float uncorrectedY, uncorrectedZ; - Merger->GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, Y, Z, uncorrectedY, uncorrectedZ); + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, Y, Z, uncorrectedY, uncorrectedZ); if (CAMath::Abs(uncorrectedY) > row.getTPCMaxY()) { return uncorrectedY; } @@ -618,7 +649,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric const float tubeMinSize2 = protect ? param.rec.tpc.tubeProtectMinSize2 : 0.f; float tubeSigma2 = protect ? param.rec.tpc.tubeProtectSigma2 : param.rec.tpc.tubeRemoveSigma2; uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, uncorrectedY)); - float time = Merger->GetConstantMem()->calibObjects.fastTransform->InverseTransformInTimeFrame(sector, uncorrectedZ + (param.par.continuousTracking ? Merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0), param.continuousMaxTimeBin); // TODO: Simplify this call in TPCFastTransform + float time = merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformInTimeFrame(sector, uncorrectedZ + (param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0), param.continuousMaxTimeBin); // TODO: Simplify this call in TPCFastTransform if (iRow < param.rec.tpc.tubeExtraProtectMinRow || pad < param.rec.tpc.tubeExtraProtectEdgePads || pad >= (uint32_t)(GPUTPCGeometry::NPads(iRow) - param.rec.tpc.tubeExtraProtectEdgePads) || param.GetUnscaledMult(time) / GPUTPCGeometry::Row2X(iRow) > param.rec.tpc.tubeExtraProtectMinOccupancy) { @@ -635,8 +666,8 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric const int32_t nBinsY = row.Grid().Ny(); const int32_t idOffset = tracker.Data().ClusterIdOffset(); const int32_t* ids = &(tracker.Data().ClusterDataIndex()[row.HitNumberOffset()]); - uint32_t myWeight = Merger->TrackOrderAttach()[iTrack] | gputpcgmmergertypes::attachAttached | gputpcgmmergertypes::attachTube; - GPUAtomic(uint32_t)* const weights = Merger->ClusterAttachment(); + uint32_t myWeight = merger.TrackOrderAttach()[iTrack] | gputpcgmmergertypes::attachAttached | gputpcgmmergertypes::attachTube; + GPUAtomic(uint32_t)* const weights = merger.ClusterAttachment(); if (goodLeg) { myWeight |= gputpcgmmergertypes::attachGoodLeg; } @@ -670,10 +701,10 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric return uncorrectedY; } -GPUd() bool GPUTPCGMTrackParam::AttachClustersPropagate(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& GPUrestrict() prop, bool inFlyDirection, float maxSinPhi, bool dodEdx) +GPUd() bool GPUTPCGMTrackParam::AttachClustersPropagate(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& GPUrestrict() prop, bool inFlyDirection, float maxSinPhi, bool dodEdx) { static constexpr float kSectAngle = 2 * M_PI / 18.f; - if (Merger->Param().rec.tpc.disableRefitAttachment & 2) { + if (merger.Param().rec.tpc.disableRefitAttachment & 2) { return dodEdx; } if (CAMath::Abs(lastRow - toRow) < 2) { @@ -694,14 +725,14 @@ GPUd() bool GPUTPCGMTrackParam::AttachClustersPropagate(const GPUTPCGMMerger* GP } if (dodEdx && iRow + step == toRow) { float yUncorrected, zUncorrected; - Merger->GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, yUncorrected)); - if (pad >= GPUTPCGeometry::NPads(iRow) || (Merger->GetConstantMem()->calibObjects.dEdxCalibContainer && Merger->GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad))) { + if (pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad))) { dodEdx = false; } } CADEBUG(printf("Attaching in row %d\n", iRow)); - AttachClusters(Merger, sector, iRow, iTrack, goodLeg, prop); + AttachClusters(merger, sector, iRow, iTrack, goodLeg, prop); } return dodEdx; } @@ -719,7 +750,7 @@ GPUdii() void GPUTPCGMTrackParam::StoreOuter(gputpcgmmergertypes::GPUTPCOuterPar outerParam->alpha = alpha; } -GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha) +GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha) { if (iRow == 0 || iRow == GPUTPCGeometry::NROWS - 1) { return; @@ -734,10 +765,10 @@ GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger* return; } - uint32_t nLoopData = CAMath::AtomicAdd(&Merger->Memory()->nLoopData, 1u); - if (nLoopData >= Merger->NMaxTracks()) { - Merger->raiseError(GPUErrors::ERROR_MERGER_LOOPER_OVERFLOW, nLoopData, Merger->NMaxTracks()); - CAMath::AtomicExch(&Merger->Memory()->nLoopData, Merger->NMaxTracks()); + uint32_t nLoopData = CAMath::AtomicAdd(&merger.Memory()->nLoopData, 1u); + if (nLoopData >= merger.NMaxTracks()) { + merger.raiseError(GPUErrors::ERROR_MERGER_LOOPER_OVERFLOW, nLoopData, merger.NMaxTracks()); + CAMath::AtomicExch(&merger.Memory()->nLoopData, merger.NMaxTracks()); return; } GPUTPCGMLoopData data; @@ -747,36 +778,36 @@ GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger* data.sector = sector; data.row = iRow; data.outwards = outwards; - Merger->LoopData()[nLoopData] = data; + merger.LoopData()[nLoopData] = data; } -GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t loopIdx) +GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t loopIdx) { GPUTPCGMPropagator prop; prop.SetMaterialTPC(); - prop.SetPolynomialField(&Merger->Param().polynomialField); + prop.SetPolynomialField(&merger.Param().polynomialField); prop.SetMaxSinPhi(constants::MAX_SIN_PHI); - prop.SetMatLUT(Merger->Param().rec.useMatLUT ? Merger->GetConstantMem()->calibObjects.matLUT : nullptr); + prop.SetMatLUT(merger.Param().rec.useMatLUT ? merger.GetConstantMem()->calibObjects.matLUT : nullptr); prop.SetSeedingErrors(false); prop.SetFitInProjections(true); prop.SetPropagateBzOnly(false); - GPUTPCGMLoopData& data = Merger->LoopData()[loopIdx]; + GPUTPCGMLoopData& data = merger.LoopData()[loopIdx]; prop.SetTrack(&data.param, data.alpha); if (false) { - data.param.AttachClustersLooper(Merger, data.sector, data.row, data.track, data.outwards, prop); + data.param.AttachClustersLooper(merger, data.sector, data.row, data.track, data.outwards, prop); } else { - data.param.AttachClustersLooperFollow(Merger, prop, data.sector, data.track, data.outwards); + data.param.AttachClustersLooperFollow(merger, prop, data.sector, data.track, data.outwards); } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger* GPUrestrict() Merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, int32_t iTrack, bool up) +GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, int32_t iTrack, bool up) { float toX = mX; - bool inFlyDirection = (Merger->MergedTracks()[iTrack].Leg() & 1) ^ up; + bool inFlyDirection = (merger.MergedTracks()[iTrack].Leg() & 1) ^ up; static constexpr float kSectAngle = 2 * M_PI / 18.f; - const GPUParam& GPUrestrict() param = Merger->Param(); + const GPUParam& GPUrestrict() param = merger.Param(); bool right = (mP[2] < 0) ^ up; const int32_t sectorSide = sector >= (int32_t)(GPUTPCGeometry::NSECTORS / 2) ? (GPUTPCGeometry::NSECTORS / 2) : 0; float lrFactor = right ^ !up ? 1.f : -1.f; @@ -813,7 +844,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger float rowX = GPUTPCGeometry::Row2X(j); if (CAMath::Abs(rowX - (-mP[0] * lrFactor)) < 1.5f) { CADEBUG(printf("\t\tAttempt row %d (X %f Y %f Z %f)\n", j, rowX, mX * lrFactor, mP[1])); - AttachClusters(Merger, sector, j, iTrack, false, mX * lrFactor, mP[1]); + AttachClusters(merger, sector, j, iTrack, false, mX * lrFactor, mP[1]); } } } @@ -839,7 +870,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& GPUrestrict() prop) +GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& GPUrestrict() prop) { static constexpr float kSectAngle = 2 * M_PI / 18.f; // Note that the coordinate system is rotated by 90 degree swapping X and Y! @@ -889,31 +920,32 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger* GPUr float rowX = mX + GPUTPCGeometry::Row2X(j) - myRowX; if (CAMath::Abs(rowX - paramX) < 1.5f) { // printf("Attempt row %d at y %f\n", j, X); - AttachClusters(Merger, sector, j, iTrack, false, mP[2] > 0 ? X : -X, Z); + AttachClusters(merger, sector, j, iTrack, false, mP[2] > 0 ? X : -X, Z); } } } } -GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, int32_t N) +GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, int32_t N) { if (N == 0) { N = 1; } - const auto& GPUrestrict() cls = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear; + const auto& GPUrestrict() cls = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear; float z0 = cls[clusters[0].num].getTime(), zn = cls[clusters[N - 1].num].getTime(); const auto tmp = zn > z0 ? std::array{zn, z0, GPUTPCGeometry::Row2X(clusters[N - 1].row)} : std::array{z0, zn, GPUTPCGeometry::Row2X(clusters[0].row)}; return ShiftZ(merger, clusters[0].sector, tmp[0], tmp[1], tmp[2]); } -GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMerger* GPUrestrict() merger, uint32_t sector, float cltmax, float cltmin, float clx) +GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMerger& GPUrestrict() merger, uint32_t sector, float cltmax, float cltmin, float clx) { - if (!merger->Param().par.continuousTracking) { + const GPUParam& GPUrestrict() param = merger.Param(); + if (!param.par.continuousTracking) { return 0.f; } float deltaZ = 0.f; bool beamlineReached = false; - const float r1 = CAMath::Max(0.0001f, CAMath::Abs(mP[4] * merger->Param().polynomialField.GetNominalBz())); + const float r1 = CAMath::Max(0.0001f, CAMath::Abs(mP[4] * param.polynomialField.GetNominalBz())); if (r1 < 0.01501) { // 100 MeV @ 0.5T ~ 0.66m cutof const float dist2 = mX * mX + mP[0] * mP[0]; const float dist1r2 = dist2 * r1 * r1; @@ -940,16 +972,16 @@ GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMerger* GPUrestrict() merg } if (!beamlineReached) { - float refZ = ((sector < GPUTPCGeometry::NSECTORS / 2) ? merger->Param().rec.tpc.defaultZOffsetOverR : -merger->Param().rec.tpc.defaultZOffsetOverR) * clx; + float refZ = ((sector < GPUTPCGeometry::NSECTORS / 2) ? param.rec.tpc.defaultZOffsetOverR : -param.rec.tpc.defaultZOffsetOverR) * clx; float basez; - merger->GetConstantMem()->calibObjects.fastTransform->TransformIdealZ(sector, cltmax, basez, mTOffset); + merger.GetConstantMem()->calibObjects.fastTransform->TransformIdealZ(sector, cltmax, basez, mTOffset); deltaZ = basez - refZ; } { - float deltaT = merger->GetConstantMem()->calibObjects.fastTransform->convDeltaZtoDeltaTimeInTimeFrame(sector, deltaZ); + float deltaT = merger.GetConstantMem()->calibObjects.fastTransform->convDeltaZtoDeltaTimeInTimeFrame(sector, deltaZ); mTOffset += deltaT; - const float maxT = cltmin - merger->GetConstantMem()->calibObjects.fastTransform->getT0(); - const float minT = cltmax - merger->GetConstantMem()->calibObjects.fastTransform->getMaxDriftTime(sector); + const float maxT = cltmin - merger.GetConstantMem()->calibObjects.fastTransform->getT0(); + const float minT = cltmax - merger.GetConstantMem()->calibObjects.fastTransform->getMaxDriftTime(sector); // printf("T Check: Clusters %f %f, min %f max %f vtx %f\n", tz1, tz2, minT, maxT, mTOffset); deltaT = 0.f; if (mTOffset < minT) { @@ -959,7 +991,7 @@ GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMerger* GPUrestrict() merg deltaT = maxT - mTOffset; } if (deltaT != 0.f) { - deltaZ += merger->GetConstantMem()->calibObjects.fastTransform->convDeltaTimeToDeltaZinTimeFrame(sector, deltaT); + deltaZ += merger.GetConstantMem()->calibObjects.fastTransform->convDeltaTimeToDeltaZinTimeFrame(sector, deltaT); // printf("Moving clusters to TPC Range: QPt %f, New mTOffset %f, t1 %f, t2 %f, Shift %f in Z: %f to %f --> %f to %f in T\n", mP[4], mTOffset + deltaT, tz1, tz2, deltaZ, tz2 - mTOffset, tz1 - mTOffset, tz2 - mTOffset - deltaT, tz1 - mTOffset - deltaT); mTOffset += deltaT; } @@ -1001,7 +1033,7 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const return ok; } -GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger, bool rebuilt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. +GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger& GPUrestrict() merger, bool rebuilt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. { if (!track.OK()) { return; @@ -1015,27 +1047,27 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, track, rebuilt); - CADEBUG(if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); + bool ok = t.Fit(merger, iTrk, nTrackHits, NTolerated, Alpha, track, rebuilt); + CADEBUG(if (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); if (CAMath::Abs(t.QPt()) < 1.e-4f) { t.QPt() = CAMath::Copysign(1.e-4f, t.QPt()); } - CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger->MergedTracks()[iTrk].Looper()); }); + CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger.MergedTracks()[iTrk].Looper()); }); - track.SetOK(ok); // TODO: Should we recover tracks who failed the fit in iWay0/1 for the rebuild? - if (t.GetNDF() <= 0 && !rebuilt && merger->Param().rec.tpc.rebuildTrackInFit) { // TODO: Better handling of NDF<0 tracks, how do we want to do cluster rejection? + track.SetOK(ok); // TODO: Should we recover tracks who failed the fit in iWay0/1 for the rebuild? + if (t.GetNDF() <= 0 && !rebuilt && merger.Param().rec.tpc.rebuildTrackInFit) { // TODO: Better handling of NDF<0 tracks, how do we want to do cluster rejection? track.Param().NDF() = 0; } else { track.Param() = t; track.Alpha() = Alpha; } - if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) { + if (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) { track.SetNClustersFitted(nTrackHits); } - // if (track.OK()) merger->DebugRefitMergedTrack(track); + // if (track.OK()) merger.DebugRefitMergedTrack(track); } GPUd() void GPUTPCGMTrackParam::Rotate(float alpha) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index ce4ab8f463d76..b149689df8a75 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -27,8 +27,14 @@ class AliExternalTrackParam; +namespace o2::tpc +{ +struct ClusterNative; +}; + namespace o2::gpu { +class GPUTPCTracker; class GPUTPCGMMerger; class GPUTPCGMBorderTrack; struct GPUParam; @@ -36,6 +42,13 @@ class GPUTPCGMPhysicalTrackModel; class GPUTPCGMPolynomialField; class GPUTPCGMMergedTrack; class GPUTPCGMPropagator; +class GPUdEdx; +struct GPUTPCGMMergedTrackHit; + +namespace gputpcgmmergertypes +{ +struct InterpolationErrorHit; +} // namespace gputpcgmmergertypes /** * @class GPUTPCGMTrackParam @@ -141,23 +154,26 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); - GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger, bool rebuilt); + GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); + GPUd() void FitdEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, const GPUParam& param, bool finalFit, const GPUCalibObjectsConst& calib, int ihit, int ihitMergeFirst, int wayDirection, const o2::tpc::ClusterNative* clustersArray, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, int32_t& N, int32_t& NTolerated, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, uint8_t& lastRow, uint8_t& lastSector, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, int32_t& ihitStart, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float& covYYUpd, int32_t& resetT0); + GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); - GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); - - GPUd() bool AttachClustersPropagate(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop, bool inFlyDirection, float maxSinPhi = constants::MAX_SIN_PHI, bool checkdEdx = false); - GPUd() float AttachClusters(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use - GPUd() float AttachClusters(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); - GPUd() void AttachClustersLooper(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& prop); - GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger* GPUrestrict() Merger, GPUTPCGMPropagator& prop, int32_t sector, int32_t iTrack, bool outwards); - GPUd() void StoreLoopPropagation(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha); + GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); + GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); + + GPUd() bool AttachClustersPropagate(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop, bool inFlyDirection, float maxSinPhi = constants::MAX_SIN_PHI, bool checkdEdx = false); + GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use + GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); + GPUd() void AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& prop); + GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() Merger, GPUTPCGMPropagator& prop, int32_t sector, int32_t iTrack, bool outwards); + GPUd() void StoreLoopPropagation(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha); GPUd() void StoreOuter(gputpcgmmergertypes::GPUTPCOuterParam* outerParam, float alpha); - GPUd() static void PropagateLooper(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t loopIdx); + GPUd() static void PropagateLooper(const GPUTPCGMMerger& merger, int32_t loopIdx); - GPUd() void AddCovDiagErrors(const float* GPUrestrict() errors2); - GPUd() void AddCovDiagErrorsWithCorrelations(const float* GPUrestrict() errors2); + GPUd() void AddCovDiagErrors(const float* errors2); + GPUd() void AddCovDiagErrorsWithCorrelations(const float* errors2); GPUdi() void MarkClusters(GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t ihitFirst, int32_t ihitLast, int32_t wayDirection, uint8_t state) { @@ -183,8 +199,8 @@ class GPUTPCGMTrackParam } GPUd() void Rotate(float alpha); - GPUd() float ShiftZ(const GPUTPCGMMerger* merger, uint32_t sector, float cltmax, float cltmin, float clx); - GPUd() float ShiftZ(const GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, int32_t N); + GPUd() float ShiftZ(const GPUTPCGMMerger& merger, uint32_t sector, float cltmax, float cltmin, float clx); + GPUd() float ShiftZ(const GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, int32_t N); GPUd() static float Reciprocal(float x) { return 1.f / x; } GPUdi() static void Assign(float& x, bool mask, float v) From c9569d86b549fc7ca734478a8e965802f2db9121 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 23 Oct 2025 13:25:01 +0200 Subject: [PATCH 08/25] GPU: Add looperFollowMode option --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index db541b0270b49..e224808a8cace 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -147,6 +147,7 @@ AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fractio AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking (prolong tracks to adjacent sectors to find short segments)") AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit (1: attachment, 2: propagation, 4: loop following)") +AddOptionRTC(looperFollowMode, uint8_t, 1, "", 0, "0 = simple, 1 = advances, for disabling use disableRefitAttachment = 4") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression (0 = no, 1 = strategy A, 2 = strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits (1=truncate charge/width LSB, 2=differences, 4=track-model)") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 5dddb2b063f57..fe5521c2b5363 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -794,10 +794,10 @@ GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestr GPUTPCGMLoopData& data = merger.LoopData()[loopIdx]; prop.SetTrack(&data.param, data.alpha); - if (false) { - data.param.AttachClustersLooper(merger, data.sector, data.row, data.track, data.outwards, prop); - } else { + if (merger.Param().rec.tpc.looperFollowMode == 1) { data.param.AttachClustersLooperFollow(merger, prop, data.sector, data.track, data.outwards); + } else { + data.param.AttachClustersLooper(merger, data.sector, data.row, data.track, data.outwards, prop); } } From 0a2e17abf75f99312fddda3490b7c1bb1025ca55 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 23 Oct 2025 13:25:20 +0200 Subject: [PATCH 09/25] GPU: simplifaction, refactoring and some minor improvements --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 2 +- GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx | 145 ++++++------- GPU/GPUTracking/Merger/GPUTPCGMPropagator.h | 12 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 196 ++++++++---------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 13 +- 5 files changed, 174 insertions(+), 194 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index e224808a8cace..936ebe3925fe3 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -141,7 +141,7 @@ AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger ( AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") -AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits, <0: reject at max -n hits") +AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx index 974a0b3ea2074..49e0773abab1d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx @@ -608,7 +608,7 @@ GPUd() float GPUTPCGMPropagator::PredictChi2(float posY, float posZ, float err2Y } } -GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, const GPUParam& GPUrestrict() param, int16_t clusterState, int8_t rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge) +GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, const GPUParam& GPUrestrict() param, int16_t clusterState, bool rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge) { float err2Y, err2Z; GetErr2(err2Y, err2Z, param, posZ, iRow, clusterState, sector, time, avgInvCharge, invCharge); @@ -616,7 +616,7 @@ GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, return Update(posY, posZ, iRow, param, clusterState, rejectChi2, refit, err2Y, err2Z); } -GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, const GPUParam& GPUrestrict() param, int16_t clusterState, int8_t rejectChi2, bool refit, float err2Y, float err2Z) +GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, const GPUParam& GPUrestrict() param, int16_t clusterState, bool rejectChi2, bool refit, float err2Y, float err2Z) { if (mT->NDF() == -5) { // first measurement: no need to filter, as the result is known in advance. just set it. mT->ResetCovariance(); @@ -635,80 +635,83 @@ GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, return 0; } - return Update(posY, posZ, clusterState, rejectChi2 == rejectDirect, err2Y, err2Z, ¶m); + return Update(posY, posZ, clusterState, rejectChi2, err2Y, err2Z, ¶m); } -GPUd() int32_t GPUTPCGMPropagator::InterpolateReject(const GPUParam& GPUrestrict() param, float posY, float posZ, int16_t clusterState, int8_t rejectChi2, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ) +GPUd() void GPUTPCGMPropagator::InterpolateFill(float posY, float posZ, gputpcgmmergertypes::InterpolationErrorHit* inter) { float* GPUrestrict() mC = mT->Cov(); float* GPUrestrict() mP = mT->Par(); - if (rejectChi2 == rejectInterFill) { - inter->posY = mP[0]; - inter->posZ = mP[1]; - if (mT->NDF() <= 0) { - inter->errorY = inter->errorZ = 100.f; - } else { - inter->errorY = mC[0]; - inter->errorZ = mC[2]; - } - } else if (rejectChi2 == rejectInterReject) { - float chi2Y, chi2Z; - if (mT->NDF() <= 0) { - chi2Y = CAMath::Square((float)inter->posY - posY) / ((float)inter->errorY + err2Y); - chi2Z = CAMath::Square((float)inter->posZ + deltaZ - posZ) / ((float)inter->errorZ + err2Z); - } else if (mFitInProjections) { - const float Iz0 = inter->posY - mP[0]; - const float Iz1 = inter->posZ + deltaZ - mP[1]; - const float Iw0 = 1.f / (mC[0] + (float)inter->errorY); - const float Iw2 = 1.f / (mC[2] + (float)inter->errorZ); - const float Ik00 = mC[0] * Iw0; - const float Ik11 = mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0; - const float ImP1 = mP[1] + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0]; - const float ImC2 = mC[2] - Ik11 * mC[2]; - // printf("\t%21sInterpo ----- abde artaf%16s Y %8.3f, Z %8.3f (Errors %f <-- (%f, %f) %f <-- (%f, %f))\n", "", "", ImP0, ImP1, sqrtf(ImC0), sqrtf(mC[0]), sqrtf(inter->errorY), sqrtf(ImC2), sqrtf(mC[2]), sqrtf(inter->errorZ)); - const float Jz0 = posY - ImP0; - const float Jz1 = posZ - ImP1; - const float Jw0 = 1.f / (ImC0 + err2Y); - const float Jw2 = 1.f / (ImC2 + err2Z); - chi2Y = Jw0 * Jz0 * Jz0; - chi2Z = Jw2 * Jz1 * Jz1; - } else { - const float Iz0 = inter->posY - mP[0]; - const float Iz1 = inter->posZ + deltaZ - mP[1]; - float Iw0 = mC[2] + (float)inter->errorZ; - float Iw2 = mC[0] + (float)inter->errorY; - float Idet = CAMath::Max(1e-10f, Iw0 * Iw2 - mC[1] * mC[1]); - Idet = 1.f / Idet; - Iw0 *= Idet; - const float Iw1 = mC[1] * Idet; - Iw2 *= Idet; - const float Ik00 = mC[0] * Iw0 + mC[1] * Iw1; - const float Ik01 = mC[0] * Iw1 + mC[1] * Iw2; - const float Ik10 = mC[1] * Iw0 + mC[2] * Iw1; - const float Ik11 = mC[1] * Iw1 + mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0 + Ik01 * Iz1; - const float ImP1 = mP[1] + Ik10 * Iz0 + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0] + Ik01 * mC[1]; - const float ImC1 = mC[1] - Ik10 * mC[0] + Ik11 * mC[1]; - const float ImC2 = mC[2] - Ik10 * mC[1] + Ik11 * mC[2]; - const float Jz0 = posY - ImP0; - const float Jz1 = posZ - ImP1; - float Jw0 = ImC2 + err2Z; - float Jw2 = ImC0 + err2Y; - float Jdet = CAMath::Max(1e-10f, Jw0 * Jw2 - ImC1 * ImC1); - Jdet = 1.f / Jdet; - Jw0 *= Jdet; - const float Jw1 = ImC1 * Jdet; - Jw2 *= Jdet; - chi2Y = CAMath::Abs((Jw0 * Jz0 + Jw1 * Jz1) * Jz0); - chi2Z = CAMath::Abs((Jw1 * Jz0 + Jw2 * Jz1) * Jz1); - } - if (RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clusterState)) { // TODO: Relative Pt resolution decreases slightly, why? - // printf("Reject Cluster chiy2 %f chiz2 %f (Pos Y: %f - %f %f ; Pos Z: %f - %f %f)\n", chi2Y, chi2Z, posY, mP[0], (float)inter->posY, posZ, mP[1], (float)inter->posZ + deltaZ); - return updateErrorClusterRejectedInInterpolation; - } + inter->posY = mP[0]; + inter->posZ = mP[1]; + if (mT->NDF() <= 0) { + inter->errorY = inter->errorZ = 100.f; + } else { + inter->errorY = mC[0]; + inter->errorZ = mC[2]; + } +} + +GPUd() int32_t GPUTPCGMPropagator::InterpolateReject(const GPUParam& GPUrestrict() param, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ) +{ + float* GPUrestrict() mC = mT->Cov(); + float* GPUrestrict() mP = mT->Par(); + float chi2Y, chi2Z; + if (mT->NDF() <= 0) { + chi2Y = CAMath::Square((float)inter->posY - posY) / ((float)inter->errorY + err2Y); + chi2Z = CAMath::Square((float)inter->posZ + deltaZ - posZ) / ((float)inter->errorZ + err2Z); + } else if (mFitInProjections) { + const float Iz0 = inter->posY - mP[0]; + const float Iz1 = inter->posZ + deltaZ - mP[1]; + const float Iw0 = 1.f / (mC[0] + (float)inter->errorY); + const float Iw2 = 1.f / (mC[2] + (float)inter->errorZ); + const float Ik00 = mC[0] * Iw0; + const float Ik11 = mC[2] * Iw2; + const float ImP0 = mP[0] + Ik00 * Iz0; + const float ImP1 = mP[1] + Ik11 * Iz1; + const float ImC0 = mC[0] - Ik00 * mC[0]; + const float ImC2 = mC[2] - Ik11 * mC[2]; + // printf("\t%21sInterpo ----- abde artaf%16s Y %8.3f, Z %8.3f (Errors %f <-- (%f, %f) %f <-- (%f, %f))\n", "", "", ImP0, ImP1, sqrtf(ImC0), sqrtf(mC[0]), sqrtf(inter->errorY), sqrtf(ImC2), sqrtf(mC[2]), sqrtf(inter->errorZ)); + const float Jz0 = posY - ImP0; + const float Jz1 = posZ - ImP1; + const float Jw0 = 1.f / (ImC0 + err2Y); + const float Jw2 = 1.f / (ImC2 + err2Z); + chi2Y = Jw0 * Jz0 * Jz0; + chi2Z = Jw2 * Jz1 * Jz1; + } else { + const float Iz0 = inter->posY - mP[0]; + const float Iz1 = inter->posZ + deltaZ - mP[1]; + float Iw0 = mC[2] + (float)inter->errorZ; + float Iw2 = mC[0] + (float)inter->errorY; + float Idet = CAMath::Max(1e-10f, Iw0 * Iw2 - mC[1] * mC[1]); + Idet = 1.f / Idet; + Iw0 *= Idet; + const float Iw1 = mC[1] * Idet; + Iw2 *= Idet; + const float Ik00 = mC[0] * Iw0 + mC[1] * Iw1; + const float Ik01 = mC[0] * Iw1 + mC[1] * Iw2; + const float Ik10 = mC[1] * Iw0 + mC[2] * Iw1; + const float Ik11 = mC[1] * Iw1 + mC[2] * Iw2; + const float ImP0 = mP[0] + Ik00 * Iz0 + Ik01 * Iz1; + const float ImP1 = mP[1] + Ik10 * Iz0 + Ik11 * Iz1; + const float ImC0 = mC[0] - Ik00 * mC[0] + Ik01 * mC[1]; + const float ImC1 = mC[1] - Ik10 * mC[0] + Ik11 * mC[1]; + const float ImC2 = mC[2] - Ik10 * mC[1] + Ik11 * mC[2]; + const float Jz0 = posY - ImP0; + const float Jz1 = posZ - ImP1; + float Jw0 = ImC2 + err2Z; + float Jw2 = ImC0 + err2Y; + float Jdet = CAMath::Max(1e-10f, Jw0 * Jw2 - ImC1 * ImC1); + Jdet = 1.f / Jdet; + Jw0 *= Jdet; + const float Jw1 = ImC1 * Jdet; + Jw2 *= Jdet; + chi2Y = CAMath::Abs((Jw0 * Jz0 + Jw1 * Jz1) * Jz0); + chi2Z = CAMath::Abs((Jw1 * Jz0 + Jw2 * Jz1) * Jz1); + } + if (RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clusterState)) { // TODO: Relative Pt resolution decreases slightly, why? + // printf("Reject Cluster chiy2 %f chiz2 %f (Pos Y: %f - %f %f ; Pos Z: %f - %f %f)\n", chi2Y, chi2Z, posY, mP[0], (float)inter->posY, posZ, mP[1], (float)inter->posZ + deltaZ); + return updateErrorClusterRejectedInInterpolation; } return 0; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h index c3f7aac84a1a5..bf67d027f6b05 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h @@ -57,11 +57,6 @@ class GPUTPCGMPropagator updateErrorClusterRejectedInUpdate = 4, updateErrorClusterRejectedEdge = 5 }; - enum RejectChi2Mode { - rejectDirect = 1, - rejectInterFill = 2, - rejectInterReject = 3 - }; GPUdDefault() GPUTPCGMPropagator() = default; @@ -104,10 +99,11 @@ class GPUTPCGMPropagator GPUd() int32_t PropagateToXAlphaBz(float posX, float posAlpha, bool inFlyDirection); - GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, int8_t rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge); - GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, int8_t rejectChi2, bool refit, float err2Y, float err2Z); + GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, bool rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge); + GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, bool rejectChi2, bool refit, float err2Y, float err2Z); GPUd() int32_t Update(float posY, float posZ, int16_t clusterState, bool rejectChi2, float err2Y, float err2Z, const GPUParam* param = nullptr); - GPUd() int32_t InterpolateReject(const GPUParam& param, float posY, float posZ, int16_t clusterState, int8_t rejectChi2, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ); + GPUd() void InterpolateFill(float posY, float posZ, gputpcgmmergertypes::InterpolationErrorHit* inter); + GPUd() int32_t InterpolateReject(const GPUParam& param, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ); GPUd() float PredictChi2(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, uint8_t sector, float time, float avgCharge, float charge) const; GPUd() float PredictChi2(float posY, float posZ, float err2Y, float err2Z) const; GPUd() static int32_t RejectCluster(float chiY, float chiZ, uint8_t clusterState) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index fe5521c2b5363..14e12f60f1ff8 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -50,7 +50,7 @@ using namespace o2::tpc; GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { static constexpr float maxSinPhi = constants::MAX_SIN_PHI; - CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); + static constexpr float kSectAngle = 2 * M_PI / 18.f; const GPUParam& GPUrestrict() param = merger.Param(); GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); @@ -104,11 +104,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const int32_t wayDirection = (iWay & 1) ? -1 : 1; for (int32_t ihit = ihitStart; ihit >= 0 && ihit < maxN; ihit += wayDirection) { - const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (clusters[ihit].sector < 18)); - if (crossCE) { - lastSector = clusters[ihit].sector; - } - if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); if (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl)) { @@ -130,11 +125,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); } - // clang-format off CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); - // CADEBUG(if ((uint32_t)merger.GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) - // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger.GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); - // clang-format on if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { nMissed++; nMissed2++; @@ -151,33 +142,64 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } const auto& cluster = clusters[ihit]; - - // clang-format off - CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); - // clang-format on - if (!param.rec.tpc.rebuildTrackInFit && allowChangeClusters && lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { - bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; - dodEdx = AttachClustersPropagate(merger, cluster.sector, lastRow, cluster.row, iTrk, track.Leg() == 0, prop, inFlyDirection, constants::MAX_SIN_PHI, dodEdx); - if (dodEdx) { - dEdx.fillSubThreshold(lastRow - wayDirection); - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { - dEdxAlt.fillSubThreshold(lastRow - wayDirection); + if (lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { + bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; + bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); + bool doIntertpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); + if (dodEdx || doAttach || doIntertpolate) { + int32_t step = cluster.row > lastRow ? 1 : -1; + float tmpX, tmpY, tmpZ; + for (int32_t iRow = lastRow + step; iRow != cluster.row; iRow += step) { + if (CAMath::Abs(mP[2]) > maxSinPhi || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(kSectAngle / 2.f)) { + break; + } + if (prop.GetPropagatedYZ(mX - GPUTPCGeometry::Row2X(iRow - step) + GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { + break; + } + merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoX(cluster.sector, iRow, tmpY, tmpZ, tmpX); + if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { + break; + } + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { + if (dodEdx) { + float yUncorrected, zUncorrected; + merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(cluster.sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); + uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(cluster.sector, iRow, yUncorrected)); + if (!(pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(cluster.sector, iRow, pad)))) { + dEdx.fillSubThreshold(iRow); + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { + dEdxAlt.fillSubThreshold(iRow); + } + } + } + } + if (doAttach) { + AttachClusters(merger, cluster.sector, iRow, iTrk, track.Leg() == 0, prop); } } } } - auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; + CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); - int32_t retValHit = FitHit(merger, iTrk, track, N, NTolerated, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, lastRow, lastSector, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, ihitStart, allowChangeClusters, refit, finalFit, nMissed, nMissed2, covYYUpd, resetT0); - if (retValHit == 1) { + auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; + const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (clusters[ihit].sector < 18)); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0); + if (retValHit == 0) { + ihitStart = ihit; + N++; + covYYUpd = mC[0]; + } else if (retValHit == 1) { break; - } else if (retValHit == -1) { + } else if (retValHit == 2) { + NTolerated++; continue; } + + lastRow = cluster.row; + lastSector = cluster.sector; } - if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255 && lastSector != 255) { + if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255) { StoreLoopPropagation(merger, lastSector, lastRow, iTrk, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastRow %d row %d out %d\n", iTrk, (int)lastRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } @@ -221,7 +243,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ return true; } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, int32_t& N, int32_t& NTolerated, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, uint8_t& lastRow, uint8_t& lastSector, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, int32_t& ihitStart, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float& covYYUpd, int32_t& resetT0) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -238,26 +260,10 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); } } - if (retValProp == 0) { - lastRow = cluster.row; - lastSector = cluster.sector; - } // clang-format off CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); // clang-format on - if (crossCE) { - if (param.rec.tpc.addErrorsCECrossing) { - if (param.rec.tpc.addErrorsCECrossing >= 2) { - AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); - } else { - AddCovDiagErrors(param.rec.tpc.errorsCECrossing); - } - } else if (mC[2] < 0.5f) { - mC[2] = 0.5f; - } - } - float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); @@ -269,33 +275,33 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { - return 1; + return 1; // bad chi2 for the whole track, stop the fit } if (retValProp || sinPhiErr) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; - NTolerated++; CADEBUG(printf(", %d --- break\n", (int32_t)sinPhiErr)); - return -1; + return 2; // Propagate failed or high incl angle } CADEBUG(printf("\n")); + if (crossCE) { + if (param.rec.tpc.addErrorsCECrossing) { + if (param.rec.tpc.addErrorsCECrossing >= 2) { + AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); + } else { + AddCovDiagErrors(param.rec.tpc.errorsCECrossing); + } + } else if (mC[2] < 0.5f) { + mC[2] = 0.5f; + } + } + int32_t retValUpd = 0, retValInt = 0; float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; } else { - int8_t rejectChi2 = 0; - if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { - if (iWay == nWays - 3) { - rejectChi2 = GPUTPCGMPropagator::rejectInterFill; - } else if (iWay == nWays - 2) { - rejectChi2 = GPUTPCGMPropagator::rejectInterReject; - } else if (iWay == nWays - 1) { - rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; - } - } - float err2Y, err2Z; const float time = merger.GetConstantMem()->ioPtrs.clustersNative ? merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].getTime() : -1.f; const float invSqrtCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? CAMath::InvSqrt(merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; @@ -305,12 +311,23 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); - if (rejectChi2 >= GPUTPCGMPropagator::rejectInterFill) { - if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - rejectChi2 = GPUTPCGMPropagator::rejectDirect; - } else { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &inter, err2Y, err2Z, deltaZ); + bool rejectChi2 = false; + if (param.rec.tpc.mergerInterpolateErrors) { + if (iWay == nWays - 3) { + prop.InterpolateFill(yy, zz, &inter); + } else if (iWay == nWays - 2) { + if (inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + rejectChi2 = true; + } else if (!param.rec.tpc.rebuildTrackInFit) { + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + } + } else if (iWay == nWays - 1) { + if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { + rejectChi2 = true; + } } + } else { + rejectChi2 = GetNDF() > 0 && allowChangeClusters; } if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { // uncorrectedY > -1e6f implies allowChangeClusters @@ -331,11 +348,8 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger if (!retValUpd && !retValInt) // track is updated { lastUpdateX = mX; - covYYUpd = mC[0]; nMissed = nMissed2 = 0; UnmarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - N++; - ihitStart = ihit; float dy = mP[0] - prop.Model().Y(); float dz = mP[1] - prop.Model().Z(); if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && dy * dy + dz * dz > 1) { @@ -343,6 +357,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger prop.SetTrack(this, prop.GetAlpha()); } FitdEdx(dEdx, dEdxAlt, param, finalFit, merger.GetConstantMem()->calibObjects, ihit, ihitMergeFirst, wayDirection, merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear, clusters, clusterState, zz); + return 0; // ok } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track if (retValInt || allowChangeClusters) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); @@ -353,10 +368,10 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger nMissed++; nMissed2++; } + return -1; // cluster rejected } else { return 1; // bad chi2 for the whole track, stop the fit } - return 0; } GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk) @@ -392,6 +407,9 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, row, ImP0, ImP1, uncorrectedY, uncorrectedZ); int32_t nCandidates = 0; + while (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates && merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + nCandidates].id > 1) { + nCandidates++; + } if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { const float kFactor = tracker.GetChiSeedFactor(); const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? @@ -701,42 +719,6 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestric return uncorrectedY; } -GPUd() bool GPUTPCGMTrackParam::AttachClustersPropagate(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& GPUrestrict() prop, bool inFlyDirection, float maxSinPhi, bool dodEdx) -{ - static constexpr float kSectAngle = 2 * M_PI / 18.f; - if (merger.Param().rec.tpc.disableRefitAttachment & 2) { - return dodEdx; - } - if (CAMath::Abs(lastRow - toRow) < 2) { - return dodEdx; - } - int32_t step = toRow > lastRow ? 1 : -1; - float xx = mX - GPUTPCGeometry::Row2X(lastRow); - for (int32_t iRow = lastRow + step; iRow != toRow; iRow += step) { - if (CAMath::Abs(mP[2]) > maxSinPhi) { - return dodEdx; - } - if (CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(kSectAngle / 2.f)) { - return dodEdx; - } - int32_t err = prop.PropagateToXAlpha(xx + GPUTPCGeometry::Row2X(iRow), prop.GetAlpha(), inFlyDirection); - if (err) { - return dodEdx; - } - if (dodEdx && iRow + step == toRow) { - float yUncorrected, zUncorrected; - merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); - uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, yUncorrected)); - if (pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad))) { - dodEdx = false; - } - } - CADEBUG(printf("Attaching in row %d\n", iRow)); - AttachClusters(merger, sector, iRow, iTrack, goodLeg, prop); - } - return dodEdx; -} - GPUdii() void GPUTPCGMTrackParam::StoreOuter(gputpcgmmergertypes::GPUTPCOuterParam* outerParam, float alpha) { CADEBUG(printf("\t%21sStorO Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f, SP %5.2f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f\n", "", alpha, mX, mP[0], mP[1], mP[4], mP[2], sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]))); @@ -801,7 +783,7 @@ GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestr } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, int32_t iTrack, bool up) +GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up) { float toX = mX; bool inFlyDirection = (merger.MergedTracks()[iTrack].Leg() & 1) ^ up; @@ -870,7 +852,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& GPUrestrict() prop) +GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& GPUrestrict() prop) { static constexpr float kSectAngle = 2 * M_PI / 18.f; // Note that the coordinate system is rotated by 90 degree swapping X and Y! @@ -880,7 +862,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUr float SinPhi = CAMath::Sqrt(1 - mP[2] * mP[2]) * (mP[2] > 0 ? -1 : 1); float b = prop.GetBz(prop.GetAlpha(), mX, mP[0], mP[1]); - float dx = outwards ? 1.f : -1.f; + float dx = up ? 1.f : -1.f; const float myRowX = GPUTPCGeometry::Row2X(iRow); // printf("\nAttachMirror sector %d row %d outwards %d\n", (int)sector, (int)iRow, (int)outwards); // printf("X %f Y %f Z %f SinPhi %f -->\n", mX, mP[0], mP[1], mP[2]); @@ -914,7 +896,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUr // printf("count %d: At X %f Y %f Z %f SinPhi %f\n", maxTries, mP[2] > 0 ? -Y : Y, mP[2] > 0 ? X : -X, Z, SinPhi); float paramX = mP[2] > 0 ? -Y : Y; - int32_t step = outwards ? 1 : -1; + int32_t step = up ? 1 : -1; int32_t found = 0; for (int32_t j = iRow; j >= 0 && j < (int32_t)GPUTPCGeometry::NROWS && found < 3; j += step) { float rowX = mX + GPUTPCGeometry::Row2X(j) - myRowX; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index b149689df8a75..d3b1d9b60ef42 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -156,19 +156,18 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() void FitdEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, const GPUParam& param, bool finalFit, const GPUCalibObjectsConst& calib, int ihit, int ihitMergeFirst, int wayDirection, const o2::tpc::ClusterNative* clustersArray, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, int32_t& N, int32_t& NTolerated, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, uint8_t& lastRow, uint8_t& lastSector, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, int32_t& ihitStart, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float& covYYUpd, int32_t& resetT0); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); - GPUd() bool AttachClustersPropagate(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop, bool inFlyDirection, float maxSinPhi = constants::MAX_SIN_PHI, bool checkdEdx = false); - GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use - GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); - GPUd() void AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& prop); - GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() Merger, GPUTPCGMPropagator& prop, int32_t sector, int32_t iTrack, bool outwards); - GPUd() void StoreLoopPropagation(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha); + GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use + GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); + GPUd() void AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& prop); + GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& prop, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up); + GPUd() void StoreLoopPropagation(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha); GPUd() void StoreOuter(gputpcgmmergertypes::GPUTPCOuterParam* outerParam, float alpha); GPUd() static void PropagateLooper(const GPUTPCGMMerger& merger, int32_t loopIdx); From 8af91ccd68f5bd730e7ca69133f84b809a56a6a7 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 27 Oct 2025 13:34:27 +0100 Subject: [PATCH 10/25] GPU TPC: Refactor cluster merging --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 106 ++++++++++-------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 2 +- 2 files changed, 63 insertions(+), 45 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 14e12f60f1ff8..2d4cc91c59165 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -121,26 +121,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t clusterState = clusters[ihit].state; const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; - { - const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; - merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); - } CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); - if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { + if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, param.rec.tpc.rebuildTrackInFit ? rebuilt : allowChangeClusters) == -1) { nMissed++; nMissed2++; continue; } - if (param.rec.tpc.rejectIFCLowRadiusCluster) { - const float r2 = xx * xx + yy * yy; - const float rmax = (83.5f + param.rec.tpc.sysClusErrorMinDist); - if (r2 < rmax * rmax) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); - continue; - } - } - const auto& cluster = clusters[ihit]; if (lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; @@ -156,14 +143,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (prop.GetPropagatedYZ(mX - GPUTPCGeometry::Row2X(iRow - step) + GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { break; } - merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoX(cluster.sector, iRow, tmpY, tmpZ, tmpX); + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(cluster.sector, iRow, tmpY, tmpZ, tmpX); if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { break; } if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { if (dodEdx) { float yUncorrected, zUncorrected; - merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(cluster.sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(cluster.sector, iRow, yUncorrected)); if (!(pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(cluster.sector, iRow, pad)))) { dEdx.fillSubThreshold(iRow); @@ -407,7 +394,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, row, ImP0, ImP1, uncorrectedY, uncorrectedZ); int32_t nCandidates = 0; - while (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates && merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + nCandidates].id > 1) { + while (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates && merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + nCandidates].id > 1) { nCandidates++; } if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { @@ -569,55 +556,86 @@ GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, bool rejectChi2) { + const int32_t ihitFirst = ihit; + { + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); + } if (ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector) { - float maxDistY, maxDistZ; - prop.GetErr2(maxDistY, maxDistZ, merger.Param(), zz, clusters[ihit].row, 0, clusters[ihit].sector, -1.f, 0.f, 0.f); // TODO: Use correct time, avgCharge - maxDistY = (maxDistY + mC[0]) * 20.f; - maxDistZ = (maxDistZ + mC[2]) * 20.f; - int32_t noReject = 0; // Cannot reject if simple estimation of y/z fails (extremely unlike case) + float maxDistY2, maxDistZ2; + bool noReject = false; // Cannot reject if simple estimation of y/z fails (extremely unlike case) if (CAMath::Abs(clAlpha - prop.GetAlpha()) > 1.e-4f) { noReject = prop.RotateToAlpha(clAlpha); } float projY = 0, projZ = 0; - if (noReject == 0) { + if (!noReject) { noReject |= prop.GetPropagatedYZ(xx, projY, projZ); } - float count = 0.f; - xx = yy = zz = 0.f; - clusterState = 0; - while (true) { - const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; - float clamp = cl.qTot; - float clx, cly, clz; - merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), clx, cly, clz, mTOffset); - float dy = cly - projY; - float dz = clz - projZ; - if (noReject == 0 && (dy * dy > maxDistY || dz * dz > maxDistZ)) { - CADEBUG(printf("\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY), sqrtf(maxDistZ), mP[0], projY, cly, mP[1], projZ, clz)); + prop.GetErr2(maxDistY2, maxDistZ2, merger.Param(), zz, clusters[ihit].row, 0, clusters[ihit].sector, -1.f, 0.f, 0.f); // TODO: Use correct time, avgCharge + const float kFactor = merger.GetConstantMem()->tpcTrackers[0].GetChiSeedFactor() * 4.f; + maxDistY2 = (maxDistY2 + mC[0]) * kFactor; + maxDistZ2 = (maxDistZ2 + mC[2]) * kFactor; + auto chkFunction = [clusters, rejectChi2, maxDistY2, maxDistZ2, projY, projZ, noReject](int32_t ih, float y, float z) { + float dy = y - projY; + float dz = z - projZ; + if (!noReject && (dy * dy > maxDistY2 || dz * dz > maxDistZ2)) { + CADEBUG(printf("\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2), mP[0], projY, y, mP[1], projZ, z)); if (rejectChi2) { - clusters[ihit].state |= GPUTPCGMMergedTrackHit::flagRejectDistance; + clusters[ih].state |= GPUTPCGMMergedTrackHit::flagRejectDistance; } + return false; } else { - CADEBUG(printf("\t\tMerging hit row %d X %f Y %f Z %f (dy %f, dz %f, chiY %f, chiZ %f)\n", clusters[ihit].row, clx, cly, clz, dy, dz, sqrtf(maxDistY), sqrtf(maxDistZ))); - xx += clx * clamp; // TODO: Weight in pad/time instead of XYZ + CADEBUG(printf("\t\tMerging hit row %d Y %f Z %f (dy %f, dz %f, chiY %f, chiZ %f)\n", clusters[ih].row, y, z, dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2))); + return true; + } + }; + const float tmpX = xx; + float count; + if (chkFunction(ihit, yy, zz)) { + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + const float clamp = cl.qTot; + xx *= clamp; + yy *= clamp; + zz *= clamp; + clusterState = clusters[ihit].state; + count = clamp; + } else { + xx = yy = zz = count = 0.f; + clusterState = 0; + } + do { + ihit += wayDirection; + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + const float clamp = cl.qTot; + float clx, cly, clz; + merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), clx, cly, clz, mTOffset); + if (chkFunction(ihit, cly, clz)) { + xx += clx * clamp; yy += cly * clamp; zz += clz * clamp; clusterState |= clusters[ihit].state; count += clamp; } - if (!(ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector)) { - break; - } - ihit += wayDirection; - } + } while (ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector); if (count < 0.1f) { CADEBUG(printf("\t\tNo matching cluster in double-row, skipping\n")); + xx = tmpX; return -1; } xx /= count; yy /= count; zz /= count; } + if (merger.Param().rec.tpc.rejectIFCLowRadiusCluster) { + const float r2 = xx * xx + yy * yy; + const float rmax2 = CAMath::Square(83.5f + merger.Param().rec.tpc.sysClusErrorMinDist); + if (r2 < rmax2) { + if (rejectChi2) { + MarkClusters(clusters, ihitFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); + } + return -1; + } + } return 0; } @@ -783,7 +801,7 @@ GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestr } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up) +GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, const int32_t iTrack, const bool up) { float toX = mX; bool inFlyDirection = (merger.MergedTracks()[iTrack].Leg() & 1) ^ up; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index d3b1d9b60ef42..6f058e992316f 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -166,7 +166,7 @@ class GPUTPCGMTrackParam GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); GPUd() void AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& prop); - GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& prop, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up); + GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& prop, int32_t sector, const int32_t iTrack, const bool up); GPUd() void StoreLoopPropagation(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha); GPUd() void StoreOuter(gputpcgmmergertypes::GPUTPCOuterParam* outerParam, float alpha); GPUd() static void PropagateLooper(const GPUTPCGMMerger& merger, int32_t loopIdx); From 0797d909cb4ef6eb5a567577c9a9c26b054ce8d8 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 24 Oct 2025 11:01:17 +0200 Subject: [PATCH 11/25] GPU: Change indexing of interpolated hit temporary data --- GPU/GPUTracking/DataTypes/GPUTPCGeometry.h | 5 + GPU/GPUTracking/Global/GPUErrorCodes.h | 35 ++-- GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h | 3 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 173 ++++++++++-------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 5 +- GPU/GPUTracking/Refit/GPUTrackingRefit.cxx | 5 +- 6 files changed, 129 insertions(+), 97 deletions(-) diff --git a/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h b/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h index 164f768d646ff..08f95c8bc8726 100644 --- a/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h +++ b/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h @@ -17,6 +17,9 @@ #include "GPUCommonDef.h" #include "DataFormatsTPC/Constants.h" +#ifndef GPUCA_GPUCODE_DEVICE +#include +#endif namespace o2::gpu { @@ -174,6 +177,8 @@ class GPUTPCGeometry const float v = (sector >= NSECTORS / 2) ? -z : z; return (250.f - v) * FACTOR_Z2T; // Used in compression, must remain constant at 250cm } + + GPUd() static constexpr float kSectAngle() { return 2 * M_PI / 18.f; } }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/Global/GPUErrorCodes.h b/GPU/GPUTracking/Global/GPUErrorCodes.h index 2080548c7775c..b3894cf267c17 100644 --- a/GPU/GPUTracking/Global/GPUErrorCodes.h +++ b/GPU/GPUTracking/Global/GPUErrorCodes.h @@ -35,22 +35,23 @@ GPUCA_ERROR_CODE(15, ERROR_SECTORDATA_Z_OVERFLOW, Sector, Value) GPUCA_ERROR_CODE(16, ERROR_MERGER_HIT_OVERFLOW, Value, Max) GPUCA_ERROR_CODE(17, ERROR_MERGER_REBUILD_HIT_OVERFLOW, Value, Max) GPUCA_ERROR_CODE(18, ERROR_MERGER_TRACK_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(19, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(20, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(21, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max) -GPUCA_ERROR_CODE(22, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max) -GPUCA_ERROR_CODE(23, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(24, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(25, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(25, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER -GPUCA_ERROR_CODE(26, ERROR_TPCZS_INVALID_ROW, SectorRow) // Data from invalid row is skipped -GPUCA_ERROR_CODE(27, ERROR_TPCZS_INVALID_NADC, SectorCRU, SamplesInPage, SamplesWritten) // Invalid number of ADC samples in header, existing samples were decoded -GPUCA_ERROR_CODE(28, ERROR_TPCZS_INCOMPLETE_HBF, SectorCRU, PacketCount, NextPacketCount) // Part of HBF is missing, decoding incomplete -GPUCA_ERROR_CODE(29, ERROR_TPCZS_INVALID_OFFSET, SectorEndpoint, Value, Expected) // Raw page is skipped since it contains invalid payload offset -GPUCA_ERROR_CODE(30, ERROR_TPCZS_INVALID_MAGIC_WORD, Value) // ZS header contains wrong magic word -GPUCA_ERROR_CODE(31, ERROR_TPCZS_PAGE_OVERFLOW, Position, PageEnd) // Ran out of page to decode -GPUCA_ERROR_CODE(32, ERROR_TPCZS_VERSION_MISMATCH, Value, Expected) // ZS decoder received page with wrong version -GPUCA_ERROR_CODE(33, ERROR_TPCZS_UNKNOWN, ErrorCode) // Unkown or invalid error code raised in decoder -GPUCA_ERROR_CODE(33, MAX_GPUCA_ERROR_NUMBER) +GPUCA_ERROR_CODE(19, ERROR_MERGER_INTERPOLATION_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(20, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(21, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(22, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max) +GPUCA_ERROR_CODE(23, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max) +GPUCA_ERROR_CODE(24, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(25, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(26, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(26, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER +GPUCA_ERROR_CODE(27, ERROR_TPCZS_INVALID_ROW, SectorRow) // Data from invalid row is skipped +GPUCA_ERROR_CODE(28, ERROR_TPCZS_INVALID_NADC, SectorCRU, SamplesInPage, SamplesWritten) // Invalid number of ADC samples in header, existing samples were decoded +GPUCA_ERROR_CODE(29, ERROR_TPCZS_INCOMPLETE_HBF, SectorCRU, PacketCount, NextPacketCount) // Part of HBF is missing, decoding incomplete +GPUCA_ERROR_CODE(30, ERROR_TPCZS_INVALID_OFFSET, SectorEndpoint, Value, Expected) // Raw page is skipped since it contains invalid payload offset +GPUCA_ERROR_CODE(31, ERROR_TPCZS_INVALID_MAGIC_WORD, Value) // ZS header contains wrong magic word +GPUCA_ERROR_CODE(32, ERROR_TPCZS_PAGE_OVERFLOW, Position, PageEnd) // Ran out of page to decode +GPUCA_ERROR_CODE(33, ERROR_TPCZS_VERSION_MISMATCH, Value, Expected) // ZS decoder received page with wrong version +GPUCA_ERROR_CODE(34, ERROR_TPCZS_UNKNOWN, ErrorCode) // Unkown or invalid error code raised in decoder +GPUCA_ERROR_CODE(34, MAX_GPUCA_ERROR_NUMBER) // #define GPUCA_CHECK_TPCZS_CORRUPTION diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h index 99c47b6228889..c729ab7ef5b2e 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h @@ -37,7 +37,8 @@ struct InterpolationErrorHit { }; struct InterpolationErrors { - InterpolationErrorHit hit[constants::MERGER_MAX_TRACK_CLUSTERS]; + static constexpr size_t size = constants::MERGER_MAX_TRACK_CLUSTERS; + InterpolationErrorHit hit[size]; }; struct GPUResolveSharedMemory : public GPUKernelTemplate::GPUSharedMemoryScan64 { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 2d4cc91c59165..292aa4a0203b3 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -50,7 +50,6 @@ using namespace o2::tpc; GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { static constexpr float maxSinPhi = constants::MAX_SIN_PHI; - static constexpr float kSectAngle = 2 * M_PI / 18.f; const GPUParam& GPUrestrict() param = merger.Param(); GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); @@ -62,7 +61,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ prop.SetPolynomialField(¶m.polynomialField); prop.SetMaxSinPhi(maxSinPhi); if (param.rec.tpc.mergerInterpolateErrors && !rebuilt) { - for (int32_t i = 0; i < N; i++) { + for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing size interpolation.hit[i].errorY = -1; } } @@ -70,6 +69,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const int32_t nWays = param.rec.tpc.nWays; const int32_t maxN = N; int32_t ihitStart = 0; + int32_t interpolatedStart = 0; float covYYUpd = 0.f; float deltaZ = 0.f; @@ -94,86 +94,81 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger.GetConstantMem()->calibObjects.matLUT : nullptr); prop.SetTrack(this, iWay && !rebuilt ? prop.GetAlpha() : Alpha); ConstrainSinPhi(iWay == 0 ? 0.95f : constants::MAX_SIN_PHI_LOW); - CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / kSectAngle) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); + CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / GPUTPCGeometry::kSectAngle()) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); N = 0; - uint8_t lastRow = 255; - uint8_t lastSector = 255; + uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastUpdateSector = 255; float lastUpdateX = -1; const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; - for (int32_t ihit = ihitStart; ihit >= 0 && ihit < maxN; ihit += wayDirection) { - if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { - CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); - if (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl)) { - NTolerated++; - } - if (finalOutInFit && !(clusters[ihit].state & (GPUTPCGMMergedTrackHit::flagReject | GPUTPCGMMergedTrackHit::flagHighIncl))) { - clusters[ihit].state |= GPUTPCGMMergedTrackHit::flagRejectErr; + for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart - wayDirection; ihit >= 0 && ihit < maxN; ihit += wayDirection) { + if (!param.rec.tpc.rebuildTrackInFit || rebuilt) { + if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { + CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); + if (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl)) { + NTolerated++; + } + if (finalOutInFit && !(clusters[ihit].state & (GPUTPCGMMergedTrackHit::flagReject | GPUTPCGMMergedTrackHit::flagHighIncl))) { + clusters[ihit].state |= GPUTPCGMMergedTrackHit::flagRejectErr; + } + continue; } - continue; } const bool allowChangeClusters = finalOutInFit && (nWays == 1 || ((iWay & 1) ? (ihit <= CAMath::Max(maxN / 2, maxN - 30)) : (ihit >= CAMath::Min(maxN / 2, 30)))); int32_t ihitMergeFirst = ihit; + interpolationIndex += wayDirection; uint8_t clusterState = clusters[ihit].state; const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); - if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, param.rec.tpc.rebuildTrackInFit ? rebuilt : allowChangeClusters) == -1) { - nMissed++; - nMissed2++; - continue; - } + const int32_t currentClusterStatus = MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, param.rec.tpc.rebuildTrackInFit ? rebuilt : allowChangeClusters); const auto& cluster = clusters[ihit]; - if (lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { - bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; + uint8_t dEdxSubThresholdRow = 255; + bool doInterpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); + if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { + bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastUpdateSector && currentClusterStatus == 0; bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); - bool doIntertpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); - if (dodEdx || doAttach || doIntertpolate) { - int32_t step = cluster.row > lastRow ? 1 : -1; - float tmpX, tmpY, tmpZ; - for (int32_t iRow = lastRow + step; iRow != cluster.row; iRow += step) { - if (CAMath::Abs(mP[2]) > maxSinPhi || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(kSectAngle / 2.f)) { - break; - } + if (dodEdx || doAttach || doInterpolate) { + int32_t step = cluster.row > lastPropagateRow ? 1 : -1; + for (int32_t iRow = lastPropagateRow + step; iRow != cluster.row; iRow += step) { + float tmpX, tmpY, tmpZ; if (prop.GetPropagatedYZ(mX - GPUTPCGeometry::Row2X(iRow - step) + GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { break; } merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(cluster.sector, iRow, tmpY, tmpZ, tmpX); - if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { - break; - } - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { - if (dodEdx) { - float yUncorrected, zUncorrected; - merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); - uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(cluster.sector, iRow, yUncorrected)); - if (!(pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(cluster.sector, iRow, pad)))) { - dEdx.fillSubThreshold(iRow); - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { - dEdxAlt.fillSubThreshold(iRow); - } - } - } - } - if (doAttach) { - AttachClusters(merger, cluster.sector, iRow, iTrk, track.Leg() == 0, prop); - } + FitAddRow(iRow, cluster.sector, iTrk, track, tmpX, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate); } } + interpolationIndex += (CAMath::Abs(cluster.row - lastPropagateRow) - 1) * wayDirection; + } + lastPropagateRow = cluster.row; + + if (currentClusterStatus) { + nMissed++; + nMissed2++; + if (doInterpolate) { + FitAddRow(cluster.row, cluster.sector, iTrk, track, xx, prop, inFlyDirection, merger, nullptr, false, false, true); + } + continue; } CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); - auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; - const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (clusters[ihit].sector < 18)); + if ((uint32_t)interpolationIndex >= interpolation.size) { + merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex, interpolation.size); + break; + } + auto& inter = interpolation.hit[interpolationIndex]; + const bool crossCE = lastUpdateSector != 255 && ((lastUpdateSector < 18) ^ (clusters[ihit].sector < 18)); int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0); if (retValHit == 0) { + DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); ihitStart = ihit; + interpolatedStart = interpolationIndex; N++; covYYUpd = mC[0]; } else if (retValHit == 1) { @@ -183,12 +178,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } - lastRow = cluster.row; - lastSector = cluster.sector; + lastUpdateRow = cluster.row; + lastUpdateSector = cluster.sector; + assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } - if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255) { - StoreLoopPropagation(merger, lastSector, lastRow, iTrk, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); - CADEBUG(printf("\t\tSTORING %d lastRow %d row %d out %d\n", iTrk, (int)lastRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); + if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255) { + StoreLoopPropagation(merger, lastUpdateSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); + CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } if (!(iWay & 1) && !finalFit && !track.CCE() && !track.Looper()) { deltaZ = ShiftZ(clusters, merger, maxN); @@ -230,7 +226,31 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ return true; } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0) +GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float tmpX, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate) +{ + if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { + return; + } + if (CAMath::Abs(mP[2]) > constants::MAX_SIN_PHI || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { + return; + } + const GPUParam& GPUrestrict() param = merger.Param(); + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { + if (dodEdx) { + float yUncorrected, zUncorrected; + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); + uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, yUncorrected)); + if (!(pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad)))) { + *dEdxSubThresholdRow = iRow; + } + } + } + if (doAttach) { + AttachClusters(merger, sector, iRow, iTrk, track.Leg() == 0, prop); + } +} + +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -343,8 +363,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger CADEBUG(printf("Reinit linearization\n")); prop.SetTrack(this, prop.GetAlpha()); } - FitdEdx(dEdx, dEdxAlt, param, finalFit, merger.GetConstantMem()->calibObjects, ihit, ihitMergeFirst, wayDirection, merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear, clusters, clusterState, zz); - return 0; // ok + return 0; // ok } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track if (retValInt || allowChangeClusters) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); @@ -464,9 +483,12 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre return uncorrectedY; } -GPUdii() void GPUTPCGMTrackParam::FitdEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, const GPUParam& GPUrestrict() param, bool finalFit, const GPUCalibObjectsConst& GPUrestrict() calib, int ihit, int ihitMergeFirst, int wayDirection, const ClusterNative* GPUrestrict() clustersArray, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz) +GPUdii() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, GPUTPCGMMerger& GPUrestrict() merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow) { + const GPUParam& GPUrestrict() param = merger.Param(); if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { + const GPUCalibObjectsConst& GPUrestrict() calib = merger.GetConstantMem()->calibObjects; + const ClusterNative* GPUrestrict() clustersArray = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear; if (param.dodEdxEnabled && finalFit) { // TODO: Costimize flag to remove, and option to remove double-clusters bool acc = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMask) == 0, accAlt = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMaskAlt) == 0; if (acc || accAlt) { @@ -486,10 +508,16 @@ GPUdii() void GPUTPCGMTrackParam::FitdEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& const auto& cluster = clusters[ihit]; if (acc) { dEdx.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], calib, zz, pad, relTime); + if (dEdxSubThresholdRow) { + dEdx.fillSubThreshold(dEdxSubThresholdRow); + } } if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { if (accAlt) { dEdxAlt.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], calib, zz, pad, relTime); + if (dEdxSubThresholdRow) { + dEdxAlt.fillSubThreshold(dEdxSubThresholdRow); + } } } } @@ -497,37 +525,36 @@ GPUdii() void GPUTPCGMTrackParam::FitdEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& } } -GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& Alpha) +GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& GPUrestrict() prop, const GPUParam& GPUrestrict() param, float& GPUrestrict() Alpha) { static constexpr float kDeg2Rad = M_PI / 180.f; - static constexpr float kSectAngle = 2 * M_PI / 18.f; if (param.rec.tpc.trackReferenceX <= 500) { GPUTPCGMTrackParam save = *this; float saveAlpha = Alpha; for (int32_t attempt = 0; attempt < 3; attempt++) { - float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * kSectAngle; + float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); Alpha += dAngle; if (prop.PropagateToXAlpha(param.rec.tpc.trackReferenceX, Alpha, 0)) { break; } ConstrainSinPhi(); - if (CAMath::Abs(mP[0]) <= mX * CAMath::Tan(kSectAngle / 2.f)) { + if (CAMath::Abs(mP[0]) <= mX * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { return; } } *this = save; Alpha = saveAlpha; } - if (CAMath::Abs(mP[0]) > mX * CAMath::Tan(kSectAngle / 2.f)) { - float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * kSectAngle; + if (CAMath::Abs(mP[0]) > mX * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { + float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); Rotate(dAngle); ConstrainSinPhi(); Alpha += dAngle; } } -GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector) +GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, float toY, float toZ, bool inFlyDirection, const GPUParam& GPUrestrict() param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector) { if (mirrorParameters) { prop.Mirror(inFlyDirection); @@ -554,7 +581,7 @@ GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, mChi2 = 0; } -GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, bool rejectChi2) +GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, const bool markReject) { const int32_t ihitFirst = ihit; { @@ -575,12 +602,12 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t const float kFactor = merger.GetConstantMem()->tpcTrackers[0].GetChiSeedFactor() * 4.f; maxDistY2 = (maxDistY2 + mC[0]) * kFactor; maxDistZ2 = (maxDistZ2 + mC[2]) * kFactor; - auto chkFunction = [clusters, rejectChi2, maxDistY2, maxDistZ2, projY, projZ, noReject](int32_t ih, float y, float z) { + auto chkFunction = [clusters, markReject, maxDistY2, maxDistZ2, projY, projZ, noReject](int32_t ih, float y, float z) { float dy = y - projY; float dz = z - projZ; if (!noReject && (dy * dy > maxDistY2 || dz * dz > maxDistZ2)) { CADEBUG(printf("\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2), mP[0], projY, y, mP[1], projZ, z)); - if (rejectChi2) { + if (markReject) { clusters[ih].state |= GPUTPCGMMergedTrackHit::flagRejectDistance; } return false; @@ -630,7 +657,7 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t const float r2 = xx * xx + yy * yy; const float rmax2 = CAMath::Square(83.5f + merger.Param().rec.tpc.sysClusErrorMinDist); if (r2 < rmax2) { - if (rejectChi2) { + if (markReject) { MarkClusters(clusters, ihitFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); } return -1; @@ -806,7 +833,6 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger float toX = mX; bool inFlyDirection = (merger.MergedTracks()[iTrack].Leg() & 1) ^ up; - static constexpr float kSectAngle = 2 * M_PI / 18.f; const GPUParam& GPUrestrict() param = merger.Param(); bool right = (mP[2] < 0) ^ up; const int32_t sectorSide = sector >= (int32_t)(GPUTPCGeometry::NSECTORS / 2) ? (GPUTPCGeometry::NSECTORS / 2) : 0; @@ -821,7 +847,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger CADEBUG(printf("\tRotated: X %f Y %f Z %f SinPhi %f (Alpha %f / %f)\n", mP[0], mX, mP[1], mP[2], prop.GetAlpha(), prop.GetAlpha() + CAMath::Pi() / 2.f)); uint32_t maxTries = 100; while (true) { - while (CAMath::Abs(mX) <= CAMath::Abs(mP[0]) * CAMath::Tan(kSectAngle / 2.f) + 0.1f) { + while (CAMath::Abs(mX) <= CAMath::Abs(mP[0]) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f) + 0.1f) { if (maxTries-- == 0) { return; } @@ -872,7 +898,6 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& GPUrestrict() prop) { - static constexpr float kSectAngle = 2 * M_PI / 18.f; // Note that the coordinate system is rotated by 90 degree swapping X and Y! float X = mP[2] > 0 ? mP[0] : -mP[0]; float Y = mP[2] > 0 ? -mX : mX; @@ -907,7 +932,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUr Y += dS * SinPhi + h4; Z += dS * mP[3]; SinPhi = newSinPhi; - if (CAMath::Abs(X) > CAMath::Abs(Y) * CAMath::Tan(kSectAngle / 2.f)) { + if (CAMath::Abs(X) > CAMath::Abs(Y) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { // printf("Abort, sector edge\n"); return; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 6f058e992316f..928b813f7e04a 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -155,12 +155,13 @@ class GPUTPCGMTrackParam GPUd() bool CheckCov() const; GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); - GPUd() void FitdEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, const GPUParam& param, bool finalFit, const GPUCalibObjectsConst& calib, int ihit, int ihitMergeFirst, int wayDirection, const o2::tpc::ClusterNative* clustersArray, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz); + GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0); + GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float tmpX, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); - GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); + GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use diff --git a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx index f8bac8ce83718..2923149ca5510 100644 --- a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx +++ b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx @@ -399,11 +399,10 @@ GPUd() int32_t GPUTrackingRefit::RefitTrack(T& trkX, bool outward, bool resetCov prop.SetAlpha(alpha); } else if constexpr (std::is_same_v) { static constexpr float kDeg2Rad = M_PI / 180.f; - static constexpr float kSectAngle = 2 * M_PI / 18.f; if (mPparam->rec.tpc.trackReferenceX <= 500) { if (prop->PropagateToXBxByBz(trk, mPparam->rec.tpc.trackReferenceX)) { - if (CAMath::Abs(trk.getY()) > trk.getX() * CAMath::Tan(kSectAngle / 2.f)) { - float newAlpha = trk.getAlpha() + CAMath::Round(CAMath::ATan2(trk.getY(), trk.getX()) / kDeg2Rad / 20.f) * kSectAngle; + if (CAMath::Abs(trk.getY()) > trk.getX() * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { + float newAlpha = trk.getAlpha() + CAMath::Round(CAMath::ATan2(trk.getY(), trk.getX()) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); GPUTPCGMTrackParam::NormalizeAlpha(newAlpha); trk.rotate(newAlpha) && prop->PropagateToXBxByBz(trk, mPparam->rec.tpc.trackReferenceX); } From 40dc2239d4f270d5f2c0e3bc4530c41049e199b9 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 27 Oct 2025 21:35:02 +0100 Subject: [PATCH 12/25] GPU: Move some functionality to CAMath --- GPU/Common/GPUCommonMath.h | 1 + GPU/GPUTracking/DataTypes/GPUTPCGeometry.h | 3 ++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 9 +++------ GPU/GPUTracking/Refit/GPUTrackingRefit.cxx | 3 +-- GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h | 2 +- GPU/GPUTracking/qa/GPUQA.cxx | 12 ++++++------ GPU/GPUTracking/qa/genEvents.cxx | 4 ++-- GPU/GPUTracking/qa/genEvents.h | 4 ++-- 8 files changed, 18 insertions(+), 20 deletions(-) diff --git a/GPU/Common/GPUCommonMath.h b/GPU/Common/GPUCommonMath.h index 0ff31899dec0c..197820b129bf4 100644 --- a/GPU/Common/GPUCommonMath.h +++ b/GPU/Common/GPUCommonMath.h @@ -95,6 +95,7 @@ class GPUCommonMath GPUhdni() constexpr static float Copysign(float x, float y) { return GPUCA_CHOICE(std::copysignf(x, y), copysignf(x, y), copysign(x, y)); } GPUd() constexpr static float TwoPi() { return 6.2831853f; } GPUd() constexpr static float Pi() { return 3.1415927f; } + GPUd() constexpr static float Deg2Rad() { return Pi() / 180.f; } GPUd() constexpr static float Round(float x); GPUd() constexpr static float Floor(float x) { return GPUCA_CHOICE(floorf(x), floorf(x), floor(x)); } GPUd() static uint32_t Float2UIntReint(const float& x); diff --git a/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h b/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h index 08f95c8bc8726..f85e19f731690 100644 --- a/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h +++ b/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h @@ -16,6 +16,7 @@ #define GPUTPCGEOMETRY_H #include "GPUCommonDef.h" +#include "GPUCommonMath.h" #include "DataFormatsTPC/Constants.h" #ifndef GPUCA_GPUCODE_DEVICE #include @@ -178,7 +179,7 @@ class GPUTPCGeometry return (250.f - v) * FACTOR_Z2T; // Used in compression, must remain constant at 250cm } - GPUd() static constexpr float kSectAngle() { return 2 * M_PI / 18.f; } + GPUd() static constexpr float kSectAngle() { return 2 * CAMath::Pi() / 18.f; } }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 292aa4a0203b3..a84f202f8ac70 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -278,8 +278,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); } - static constexpr float kDeg2Rad = M_PI / 180.f; - const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); + const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { return 1; // bad chi2 for the whole track, stop the fit @@ -527,13 +526,11 @@ GPUdii() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& G GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& GPUrestrict() prop, const GPUParam& GPUrestrict() param, float& GPUrestrict() Alpha) { - static constexpr float kDeg2Rad = M_PI / 180.f; - if (param.rec.tpc.trackReferenceX <= 500) { GPUTPCGMTrackParam save = *this; float saveAlpha = Alpha; for (int32_t attempt = 0; attempt < 3; attempt++) { - float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); + float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / CAMath::Deg2Rad() / 20.f) * GPUTPCGeometry::kSectAngle(); Alpha += dAngle; if (prop.PropagateToXAlpha(param.rec.tpc.trackReferenceX, Alpha, 0)) { break; @@ -547,7 +544,7 @@ GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& GPUrestric Alpha = saveAlpha; } if (CAMath::Abs(mP[0]) > mX * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { - float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); + float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / CAMath::Deg2Rad() / 20.f) * GPUTPCGeometry::kSectAngle(); Rotate(dAngle); ConstrainSinPhi(); Alpha += dAngle; diff --git a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx index 2923149ca5510..300a469c4fbbb 100644 --- a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx +++ b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx @@ -398,11 +398,10 @@ GPUd() int32_t GPUTrackingRefit::RefitTrack(T& trkX, bool outward, bool resetCov trk.NormalizeAlpha(alpha); prop.SetAlpha(alpha); } else if constexpr (std::is_same_v) { - static constexpr float kDeg2Rad = M_PI / 180.f; if (mPparam->rec.tpc.trackReferenceX <= 500) { if (prop->PropagateToXBxByBz(trk, mPparam->rec.tpc.trackReferenceX)) { if (CAMath::Abs(trk.getY()) > trk.getX() * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { - float newAlpha = trk.getAlpha() + CAMath::Round(CAMath::ATan2(trk.getY(), trk.getX()) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); + float newAlpha = trk.getAlpha() + CAMath::Round(CAMath::ATan2(trk.getY(), trk.getX()) / CAMath::Deg2Rad() / 20.f) * GPUTPCGeometry::kSectAngle(); GPUTPCGMTrackParam::NormalizeAlpha(newAlpha); trk.rotate(newAlpha) && prop->PropagateToXBxByBz(trk, mPparam->rec.tpc.trackReferenceX); } diff --git a/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h b/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h index 2b847e20de6fb..577c89502db6f 100644 --- a/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h +++ b/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h @@ -147,7 +147,7 @@ class trackInterface : public GPUTPCGMTrackParam GPUd() float getSnp() const { return GetSinPhi(); } GPUd() float getTgl() const { return GetDzDs(); } GPUd() float getQ2Pt() const { return GetQPt(); } - GPUd() float getEta() const { return -CAMath::Log(CAMath::Tan(0.5f * (0.5f * M_PI - CAMath::ATan(getTgl())))); } + GPUd() float getEta() const { return -CAMath::Log(CAMath::Tan(0.5f * (0.5f * CAMath::Pi() - CAMath::ATan(getTgl())))); } GPUd() float getPt() const { return CAMath::Abs(getQ2Pt()) > 0 ? CAMath::Abs(1.f / getQ2Pt()) : 99999.f; } GPUd() float getSigmaY2() const { return GetErr2Y(); } GPUd() float getSigmaZ2() const { return GetErr2Z(); } diff --git a/GPU/GPUTracking/qa/GPUQA.cxx b/GPU/GPUTracking/qa/GPUQA.cxx index 9f24f3745c45c..d3266e8f7c5f1 100644 --- a/GPU/GPUTracking/qa/GPUQA.cxx +++ b/GPU/GPUTracking/qa/GPUQA.cxx @@ -182,7 +182,7 @@ static const constexpr int32_t COLORS_HEX[COLORCOUNT] = {0xB03030, 0x00A000, 0x0 static const constexpr int32_t CONFIG_DASHED_MARKERS = 0; static const constexpr float AXES_MIN[5] = {-Y_MAX, -Z_MAX, 0.f, -ETA_MAX, PT_MIN}; -static const constexpr float AXES_MAX[5] = {Y_MAX, Z_MAX, 2.f * M_PI, ETA_MAX, PT_MAX}; +static const constexpr float AXES_MAX[5] = {Y_MAX, Z_MAX, 2.f * CAMath::Pi(), ETA_MAX, PT_MAX}; static const constexpr int32_t AXIS_BINS[5] = {51, 51, 144, 31, 50}; static const constexpr int32_t RES_AXIS_BINS[] = {1017, 113}; // Consecutive bin sizes, histograms are binned down until the maximum entry is 50, each bin size should evenly divide its predecessor. static const constexpr float RES_AXES[5] = {1., 1., 0.03, 0.03, 1.0}; @@ -1213,12 +1213,12 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx const mcInfo_t& info = GetMCTrack(i, iCol); additionalMCParameters& mc2 = mMCParam[iCol][i]; mc2.pt = std::sqrt(info.pX * info.pX + info.pY * info.pY); - mc2.phi = M_PI + std::atan2(-info.pY, -info.pX); + mc2.phi = CAMath::Pi() + std::atan2(-info.pY, -info.pX); float p = info.pX * info.pX + info.pY * info.pY + info.pZ * info.pZ; if (p < 1e-18) { mc2.theta = mc2.eta = 0.f; } else { - mc2.theta = info.pZ == 0 ? (M_PI / 2) : (std::acos(info.pZ / std::sqrt(p))); + mc2.theta = info.pZ == 0 ? (CAMath::Pi() / 2) : (std::acos(info.pZ / std::sqrt(p))); mc2.eta = -std::log(std::tan(0.5 * mc2.theta)); } if (mConfig.writeMCLabels) { @@ -1272,10 +1272,10 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } float alpha = std::atan2(info.y, info.x); - alpha /= M_PI / 9.f; + alpha /= CAMath::Pi() / 9.f; alpha = std::floor(alpha); - alpha *= M_PI / 9.f; - alpha += M_PI / 18.f; + alpha *= CAMath::Pi() / 9.f; + alpha += CAMath::Pi() / 18.f; float c = std::cos(alpha); float s = std::sin(alpha); diff --git a/GPU/GPUTracking/qa/genEvents.cxx b/GPU/GPUTracking/qa/genEvents.cxx index 5af7698fdb4dc..1c7b19f20bf4e 100644 --- a/GPU/GPUTracking/qa/genEvents.cxx +++ b/GPU/GPUTracking/qa/genEvents.cxx @@ -199,8 +199,8 @@ int32_t genEvents::GenerateEvent(const GPUParam& param, const char* filename) double eta = gRandom->Uniform(-1.5, 1.5); double theta = 2 * std::atan(1. / exp(eta)); - double lambda = theta - M_PI / 2; - // double theta = gRandom->Uniform(-60,60)*M_PI/180.; + double lambda = theta - CAMath::Pi() / 2; + // double theta = gRandom->Uniform(-60,60)*CAMath::Pi()/180.; double pt = .08 * std::pow(10, gRandom->Uniform(0, 2.2)); double q = 1.; diff --git a/GPU/GPUTracking/qa/genEvents.h b/GPU/GPUTracking/qa/genEvents.h index ee5510c729525..2efe7a983a320 100644 --- a/GPU/GPUTracking/qa/genEvents.h +++ b/GPU/GPUTracking/qa/genEvents.h @@ -16,7 +16,7 @@ #define GENEVENTS_H #include "GPUCommonDef.h" -#include +#include "GPUCommonMath.h" namespace o2::gpu { @@ -66,7 +66,7 @@ class genEvents uint32_t id; }; - const double mTwoPi = 2 * M_PI; + const double mTwoPi = 2 * CAMath::Pi(); const double mSectorDAngle = mTwoPi / 18.; const double mSectorAngleOffset = mSectorDAngle / 2; From 0d505a37235e735f34cc2cf8a178211be261bbcd Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 30 Oct 2025 00:11:27 +0100 Subject: [PATCH 13/25] GPU TPC: Store track position for interpolation also when not updating with new hit --- GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx | 2 +- GPU/GPUTracking/Merger/GPUTPCGMPropagator.h | 2 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 133 ++++++++++-------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 4 +- 4 files changed, 76 insertions(+), 65 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx index 49e0773abab1d..ddfefe8636af5 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx @@ -638,7 +638,7 @@ GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, return Update(posY, posZ, clusterState, rejectChi2, err2Y, err2Z, ¶m); } -GPUd() void GPUTPCGMPropagator::InterpolateFill(float posY, float posZ, gputpcgmmergertypes::InterpolationErrorHit* inter) +GPUd() void GPUTPCGMPropagator::InterpolateFill(gputpcgmmergertypes::InterpolationErrorHit* inter) { float* GPUrestrict() mC = mT->Cov(); float* GPUrestrict() mP = mT->Par(); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h index bf67d027f6b05..e637c315f3ce6 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h @@ -102,7 +102,7 @@ class GPUTPCGMPropagator GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, bool rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge); GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, bool rejectChi2, bool refit, float err2Y, float err2Z); GPUd() int32_t Update(float posY, float posZ, int16_t clusterState, bool rejectChi2, float err2Y, float err2Z, const GPUParam* param = nullptr); - GPUd() void InterpolateFill(float posY, float posZ, gputpcgmmergertypes::InterpolationErrorHit* inter); + GPUd() void InterpolateFill(gputpcgmmergertypes::InterpolationErrorHit* inter); GPUd() int32_t InterpolateReject(const GPUParam& param, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ); GPUd() float PredictChi2(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, uint8_t sector, float time, float avgCharge, float charge) const; GPUd() float PredictChi2(float posY, float posZ, float err2Y, float err2Z) const; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index a84f202f8ac70..d7f9fda9655d6 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -123,10 +123,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t clusterState = clusters[ihit].state; const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; - CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); const int32_t currentClusterStatus = MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, param.rec.tpc.rebuildTrackInFit ? rebuilt : allowChangeClusters); + // TODO: Check about tracks who have clusters in the same row multiple times in different sectors const auto& cluster = clusters[ihit]; + CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); + CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); + uint8_t dEdxSubThresholdRow = 255; bool doInterpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { @@ -140,31 +143,67 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ break; } merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(cluster.sector, iRow, tmpY, tmpZ, tmpX); - FitAddRow(iRow, cluster.sector, iTrk, track, tmpX, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate); + if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { + break; + } + FitAddRow(iRow, cluster.sector, iTrk, track, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate); } } interpolationIndex += (CAMath::Abs(cluster.row - lastPropagateRow) - 1) * wayDirection; } lastPropagateRow = cluster.row; - if (currentClusterStatus) { - nMissed++; + int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); + if ((retValProp == -2 && // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha + (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) != 0 || // Cannot rotate to new alpha at all + prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection))) || // propagation fails nonetheless + retValProp) { // failed for other reason but rotation + CADEBUG(printf(" --- break-prop\n")); + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; - if (doInterpolate) { - FitAddRow(cluster.row, cluster.sector, iTrk, track, xx, prop, inFlyDirection, merger, nullptr, false, false, true); - } + NTolerated++; continue; } - - CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); + // clang-format off + CADEBUG(if (!CheckCov()){printf("INVALID COV AFTER PROPAGATE!!!\n");}); + CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); + // clang-format on + if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { + break; // bad chi2 for the whole track, stop the fit + } if ((uint32_t)interpolationIndex >= interpolation.size) { merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex, interpolation.size); break; } auto& inter = interpolation.hit[interpolationIndex]; + if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 3) { + prop.InterpolateFill(&inter); + } + + float uncorrectedY = -1e6f; + if (param.rec.tpc.rebuildTrackInFit) { + if (iWay == nWays - 2) { + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + } + if (allowChangeClusters) { + AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); // TODO: Do this during FindBestInterpolatedHit + } + } else if (allowChangeClusters) { + uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); + } else if (param.rec.tpc.rejectEdgeClustersInTrackFit) { + float tmpZ; + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, mP[0], mP[1], uncorrectedY, tmpZ); + } + + if (currentClusterStatus) { + nMissed++; + nMissed2++; + continue; + } + const bool crossCE = lastUpdateSector != 255 && ((lastUpdateSector < 18) ^ (clusters[ihit].sector < 18)); - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); ihitStart = ihit; @@ -226,11 +265,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ return true; } -GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float tmpX, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate) +GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate) { - if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { - return; - } if (CAMath::Abs(mP[2]) > constants::MAX_SIN_PHI || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { return; } @@ -250,46 +286,20 @@ GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t se } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; const int32_t wayDirection = (iWay & 1) ? -1 : 1; const auto& cluster = clusters[ihit]; - int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); - // clang-format off - CADEBUG(if (!CheckCov()){printf("INVALID COV AFTER PROPAGATE!!!\n");}); - // clang-format on - if (retValProp == -2) // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha - { - CADEBUG(printf("REROTATE\n")); - if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) == 0) { - retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); - } - } - // clang-format off - CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); - // clang-format on - - float uncorrectedY = -1e6f; - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { - uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); - } else if (allowChangeClusters) { - uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); - } const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); - const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; - if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { - return 1; // bad chi2 for the whole track, stop the fit - } - if (retValProp || sinPhiErr) { + if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; - CADEBUG(printf(", %d --- break\n", (int32_t)sinPhiErr)); + CADEBUG(printf(" --- break-sinphi\n")); return 2; // Propagate failed or high incl angle } - CADEBUG(printf("\n")); if (crossCE) { if (param.rec.tpc.addErrorsCECrossing) { @@ -307,6 +317,9 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; + if (param.rec.tpc.rebuildTrackInFit) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); // Will enable rejectChi2 in further rounds + } } else { float err2Y, err2Z; const float time = merger.GetConstantMem()->ioPtrs.clustersNative ? merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].getTime() : -1.f; @@ -317,15 +330,15 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); - bool rejectChi2 = false; + bool rejectChi2 = (clusterState & GPUTPCGMMergedTrackHit::flagReject); if (param.rec.tpc.mergerInterpolateErrors) { - if (iWay == nWays - 3) { - prop.InterpolateFill(yy, zz, &inter); - } else if (iWay == nWays - 2) { - if (inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - rejectChi2 = true; - } else if (!param.rec.tpc.rebuildTrackInFit) { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + if (iWay == nWays - 2) { + if (!param.rec.tpc.rebuildTrackInFit) { + if (inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + rejectChi2 = true; + } else { + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + } } } else if (iWay == nWays - 1) { if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { @@ -333,10 +346,10 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } } } else { - rejectChi2 = GetNDF() > 0 && allowChangeClusters; + rejectChi2 = allowChangeClusters; } - if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { // uncorrectedY > -1e6f implies allowChangeClusters + if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedEdge; } else { retValUpd = prop.Update(yy, zz, cluster.row, param, clusterState, rejectChi2, refit, err2Y, err2Z); @@ -367,7 +380,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger if (retValInt || allowChangeClusters) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); } else if (finalFit) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected ? GPUTPCGMMergedTrackHit::flagRejectDistance : GPUTPCGMMergedTrackHit::flagRejectErr); } if (!retValInt) { nMissed++; @@ -473,7 +486,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre } } } - CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("iTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); + CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("\t\t\tiTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); } if (nCandidates == 0) { merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; @@ -599,17 +612,17 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t const float kFactor = merger.GetConstantMem()->tpcTrackers[0].GetChiSeedFactor() * 4.f; maxDistY2 = (maxDistY2 + mC[0]) * kFactor; maxDistZ2 = (maxDistZ2 + mC[2]) * kFactor; - auto chkFunction = [clusters, markReject, maxDistY2, maxDistZ2, projY, projZ, noReject](int32_t ih, float y, float z) { + auto chkFunction = [clusters, markReject, maxDistY2, maxDistZ2, projY, projZ, noReject CADEBUG(, this)](int32_t ih, float y, float z) { float dy = y - projY; float dz = z - projZ; if (!noReject && (dy * dy > maxDistY2 || dz * dz > maxDistZ2)) { - CADEBUG(printf("\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2), mP[0], projY, y, mP[1], projZ, z)); + CADEBUG(printf("\t\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2), mP[0], projY, y, mP[1], projZ, z)); if (markReject) { clusters[ih].state |= GPUTPCGMMergedTrackHit::flagRejectDistance; } return false; } else { - CADEBUG(printf("\t\tMerging hit row %d Y %f Z %f (dy %f, dz %f, chiY %f, chiZ %f)\n", clusters[ih].row, y, z, dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2))); + CADEBUG(printf("\t\t\tMerging hit row %d Y %f Z %f (dy %f, dz %f, chiY %f, chiZ %f)\n", clusters[ih].row, y, z, dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2))); return true; } }; @@ -1061,9 +1074,7 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() return; } - // clang-format off - CADEBUG(if (DEBUG_SINGLE_TRACK != -1 && iTrk != ((DEBUG_SINGLE_TRACK == -2 && getenv("DEBUG_TRACK")) ? atoi(getenv("DEBUG_TRACK")) : DEBUG_SINGLE_TRACK)) { track.SetNClusters(0); track.SetOK(0); return; } ); - // clang-format on + CADEBUG(if (DEBUG_SINGLE_TRACK != -1 && iTrk != ((DEBUG_SINGLE_TRACK == -2 && getenv("DEBUG_TRACK")) ? atoi(getenv("DEBUG_TRACK")) : DEBUG_SINGLE_TRACK)) { track.SetNClusters(0); track.SetOK(0); return; }); int32_t nTrackHits = track.NClusters(); int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 928b813f7e04a..6e85168d2de55 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -156,8 +156,8 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0); - GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float tmpX, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); + GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); From 3b2b4f423c5f33b41c213e77f1173a8488d99fd2 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 30 Oct 2025 10:39:27 +0100 Subject: [PATCH 14/25] GPU TPC: Extract CE crossing handling in function, and apply before storing interpolated position --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 50 +++++++++++-------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 3 +- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index d7f9fda9655d6..1afd4306c274b 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -97,7 +97,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / GPUTPCGeometry::kSectAngle()) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); N = 0; - uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastUpdateSector = 255; + uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastSector = 255; float lastUpdateX = -1; const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; @@ -133,7 +133,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t dEdxSubThresholdRow = 255; bool doInterpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { - bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastUpdateSector && currentClusterStatus == 0; + bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); if (dodEdx || doAttach || doInterpolate) { int32_t step = cluster.row > lastPropagateRow ? 1 : -1; @@ -177,9 +177,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ break; } auto& inter = interpolation.hit[interpolationIndex]; - if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 3) { - prop.InterpolateFill(&inter); - } float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit) { @@ -196,14 +193,20 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, mP[0], mP[1], uncorrectedY, tmpZ); } + HandleCrossCE(param, cluster.sector, lastSector); + lastSector = cluster.sector; + + if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 3) { + prop.InterpolateFill(&inter); + } + if (currentClusterStatus) { nMissed++; nMissed2++; continue; } - const bool crossCE = lastUpdateSector != 255 && ((lastUpdateSector < 18) ^ (clusters[ihit].sector < 18)); - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); ihitStart = ihit; @@ -218,11 +221,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } lastUpdateRow = cluster.row; - lastUpdateSector = cluster.sector; assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255) { - StoreLoopPropagation(merger, lastUpdateSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); + StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } if (!(iWay & 1) && !finalFit && !track.CCE() && !track.Looper()) { @@ -286,7 +288,23 @@ GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t se } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY) +GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() param, const uint8_t sector, const uint8_t& lastSector) +{ + const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (sector < 18)); + if (crossCE) { + if (param.rec.tpc.addErrorsCECrossing) { + if (param.rec.tpc.addErrorsCECrossing >= 2) { + AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); + } else { + AddCovDiagErrors(param.rec.tpc.errorsCECrossing); + } + } else if (mC[2] < 0.5f) { + mC[2] = 0.5f; + } + } +} + +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -301,18 +319,6 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger return 2; // Propagate failed or high incl angle } - if (crossCE) { - if (param.rec.tpc.addErrorsCECrossing) { - if (param.rec.tpc.addErrorsCECrossing >= 2) { - AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); - } else { - AddCovDiagErrors(param.rec.tpc.errorsCECrossing); - } - } else if (mC[2] < 0.5f) { - mC[2] = 0.5f; - } - } - int32_t retValUpd = 0, retValInt = 0; float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 6e85168d2de55..1c5efc3b22ed7 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -156,8 +156,9 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); + GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); From fa0a4a6d17f435d39f00264ece746dac5103f395 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 30 Oct 2025 11:16:51 +0100 Subject: [PATCH 15/25] GPU TPC: If value from other side of interpolation not stored, use only current value --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 3 +- GPU/GPUTracking/Global/GPUChainTracking.cxx | 1 + GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 36 +++++++++++-------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 936ebe3925fe3..ca29b992a7284 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -109,6 +109,8 @@ AddOptionRTC(rejectEdgeClustersMargin, float, 0.f, "", 0, "Margin in cm of Y pos AddOptionRTC(rejectEdgeClustersSigmaMargin, float, 0.f, "", 0, "Margin factor for trackSigmaY when rejecting edge clusters based on uncorrected track Y") AddOptionRTC(trackletMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for tracklet") AddOptionRTC(trackletMinSharedNormFactor, float, 0.f, "", 0, "Max shared defined as trackletMinSharedNormFactor*max(current_nhits,trackletMinSharedNormFactor*minHits,1)") +AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") +AddOptionRTC(rebuildTrackMaxNonIntCov, float, 4.f, "", 0, "Max Err2 allowed for non-interpolated cluster attachment during rebuild") AddOptionRTC(maxTimeBinAboveThresholdIn1000Bin, uint16_t, 500, "", 0, "Except pad from cluster finding if total number of charges in a fragment is above this baseline (disable = 0)") AddOptionRTC(maxConsecTimeBinAboveThreshold, uint16_t, 200, "", 0, "Except pad from cluster finding if number of consecutive charges in a fragment is above this baseline (disable = 0)") AddOptionRTC(noisyPadSaturationThreshold, uint16_t, 700, "", 0, "Threshold where a timebin is considered saturated, disabling the noisy pad check for that pad") @@ -140,7 +142,6 @@ AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") -AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index 3350d12664c1a..268d5f2cd5712 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -1002,6 +1002,7 @@ void GPUChainTracking::ApplySyncSettings(GPUSettingsProcessing& proc, GPUSetting { if (syncMode) { rec.useMatLUT = false; + rec.tpc.rebuildTrackMaxNonIntCov = 0.f; } if (proc.rtc.optSpecialCode == -1) { proc.rtc.optSpecialCode = syncMode; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 1afd4306c274b..583923e297dfe 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -406,7 +406,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre GPUglobalref() const cahit2* hits = tracker.HitData(rowData); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(rowData); float uncorrectedY = -1e6f, uncorrectedZ; - if (rowData.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + if (rowData.NHits() && (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0 || (param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = rowData.Grid().YMin(); const float stepY = rowData.HstepY(); @@ -414,19 +414,24 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre const float stepZ = rowData.HstepZ(); int32_t bin, ny, nz; - float err2Y, err2Z; - param.GetClusterErrors2(sector, row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge - - const float Iz0 = inter.posY - mP[0]; - const float Iz1 = inter.posZ + deltaZ - mP[1]; - const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); - const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); - const float Ik00 = mC[0] * Iw0; - const float Ik11 = mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0; - const float ImP1 = mP[1] + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0]; - const float ImC2 = mC[2] - Ik11 * mC[2]; + float ImP0, ImP1, ImC0, ImC2; + if (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + const float Iz0 = inter.posY - mP[0]; + const float Iz1 = inter.posZ + deltaZ - mP[1]; + const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); + const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); + const float Ik00 = mC[0] * Iw0; + const float Ik11 = mC[2] * Iw2; + ImP0 = mP[0] + Ik00 * Iz0; + ImP1 = mP[1] + Ik11 * Iz1; + ImC0 = mC[0] - Ik00 * mC[0]; + ImC2 = mC[2] - Ik11 * mC[2]; + } else { + ImP0 = mP[0]; + ImP1 = mP[1]; + ImC0 = mC[0]; + ImC2 = mC[2]; + } merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, row, ImP0, ImP1, uncorrectedY, uncorrectedZ); @@ -435,6 +440,9 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre nCandidates++; } if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { + float err2Y, err2Z; + param.GetClusterErrors2(sector, row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + const float kFactor = tracker.GetChiSeedFactor(); const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(mC[2]))); From 1887b722d6aa12ac1edfe88cc4b800af9acbec4d Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 31 Oct 2025 00:00:45 +0100 Subject: [PATCH 16/25] GPU TPC: Mark clusters of rebuilt track as high incli when interpolated in NoUpdateHighIncl region --- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 6 ++- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 5 ++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 37 +++++++++++++------ 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 8083702bc3e98..3c25fde41866c 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -2266,7 +2266,11 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads } const auto& best = candidates[candidates[0].best - 1]; const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[best.id - 2]; - outCl = {.num = best.id - 2, .sector = best.sector, .row = (uint8_t)j, .state = (uint8_t)(cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags)}; + uint8_t flags = (uint8_t)(cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags); + if ((mTrackRebuildHelper[i].highInclRowLow != 255 && j <= mTrackRebuildHelper[i].highInclRowLow) || (mTrackRebuildHelper[i].highInclRowHigh != 255 && j >= mTrackRebuildHelper[i].highInclRowHigh)) { + flags |= GPUTPCGMMergedTrackHit::flagHighIncl; + } + outCl = {.num = best.id - 2, .sector = best.sector, .row = (uint8_t)j, .state = flags}; written++; CADEBUG(printf("REBUILD: iTrk %d Assigned Cluster Row %d Hit %d\n", i, j, best.id - 2)); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 22dd41a6e7a5b..3d6a2fcad3ad3 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -88,7 +88,9 @@ class GPUTPCGMMerger : public GPUProcessor }; struct trackRebuildHelper { - bool reverse; + uint8_t reverse; + uint8_t highInclRowLow; + uint8_t highInclRowHigh; }; struct tmpSort { @@ -133,6 +135,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() const GPUTPCGMMergedTrackHit* Clusters() const { return mClusters; } GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return mClusters; } GPUhdi() trackCluster* ClusterCandidates() { return mClusterCandidates; } + GPUhdi() trackRebuildHelper* TrackRebuildHelper() { return mTrackRebuildHelper; } GPUhdi() int32_t* HitWeights() { return mHitWeights; } GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 583923e297dfe..82c7fa741f7d4 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -64,6 +64,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing size interpolation.hit[i].errorY = -1; } + if (param.rec.tpc.rebuildTrackInFit) { + merger.TrackRebuildHelper()[iTrk].highInclRowLow = 255; + merger.TrackRebuildHelper()[iTrk].highInclRowHigh = 255; + } } const int32_t nWays = param.rec.tpc.nWays; @@ -206,6 +210,28 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } + const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); + if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); + nMissed2++; + CADEBUG(printf(" --- break-sinphi\n")); + NTolerated++; + const bool inward = clusters[0].row > clusters[maxN - 1].row; + const bool markHighIncl = (mP[2] > 0) ^ (mP[4] < 0) ^ inward ^ (iWay & 1); + if (param.rec.tpc.rebuildTrackInFit && markHighIncl) { + if (inward ^ (iWay & 1)) { + if (merger.TrackRebuildHelper()[iTrk].highInclRowLow == 255) { + merger.TrackRebuildHelper()[iTrk].highInclRowLow = cluster.row; + } + } else { + if (merger.TrackRebuildHelper()[iTrk].highInclRowHigh == 255) { + merger.TrackRebuildHelper()[iTrk].highInclRowHigh = cluster.row; + } + } + } + continue; + } + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); @@ -215,9 +241,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ covYYUpd = mC[0]; } else if (retValHit == 1) { break; - } else if (retValHit == 2) { - NTolerated++; - continue; } lastUpdateRow = cluster.row; @@ -311,14 +334,6 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger const int32_t wayDirection = (iWay & 1) ? -1 : 1; const auto& cluster = clusters[ihit]; - const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); - if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - nMissed2++; - CADEBUG(printf(" --- break-sinphi\n")); - return 2; // Propagate failed or high incl angle - } - int32_t retValUpd = 0, retValInt = 0; float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { From bbe7d4157f398cd5e42dd664c4baf2efec7ea49d Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 30 Oct 2025 18:39:10 +0100 Subject: [PATCH 17/25] GPU TPC: Pick up best hit from interpolation only, if current position not available --- GPU/GPUTracking/Global/GPUChainTracking.cxx | 2 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 54 +++++++++++++++---- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 3 +- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index 268d5f2cd5712..f23f5fd5b5e63 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -1002,7 +1002,7 @@ void GPUChainTracking::ApplySyncSettings(GPUSettingsProcessing& proc, GPUSetting { if (syncMode) { rec.useMatLUT = false; - rec.tpc.rebuildTrackMaxNonIntCov = 0.f; + rec.tpc.rebuildTrackMaxNonIntCov = 0.f; // TODO: Check if this yields a performance benefit } if (proc.rtc.optSpecialCode == -1) { proc.rtc.optSpecialCode = syncMode; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 82c7fa741f7d4..275cf923c8d87 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -106,7 +106,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; - for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart - wayDirection; ihit >= 0 && ihit < maxN; ihit += wayDirection) { + for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart; ihit >= 0 && ihit < maxN; ihit += wayDirection, interpolationIndex += wayDirection) { if (!param.rec.tpc.rebuildTrackInFit || rebuilt) { if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); @@ -123,7 +123,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const bool allowChangeClusters = finalOutInFit && (nWays == 1 || ((iWay & 1) ? (ihit <= CAMath::Max(maxN / 2, maxN - 30)) : (ihit >= CAMath::Min(maxN / 2, 30)))); int32_t ihitMergeFirst = ihit; - interpolationIndex += wayDirection; uint8_t clusterState = clusters[ihit].state; const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; @@ -157,6 +156,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } lastPropagateRow = cluster.row; + auto& inter = interpolation.hit[interpolationIndex]; int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); if ((retValProp == -2 && // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) != 0 || // Cannot rotate to new alpha at all @@ -166,6 +166,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; NTolerated++; + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); // TODO: mark clusters from single-sided finding as dubious, and apply stricter restriction cut later + } continue; } // clang-format off @@ -173,6 +176,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); // clang-format on if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + } break; // bad chi2 for the whole track, stop the fit } @@ -180,12 +186,11 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex, interpolation.size); break; } - auto& inter = interpolation.hit[interpolationIndex]; float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit) { if (iWay == nWays - 2) { - uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); } if (allowChangeClusters) { AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); // TODO: Do this during FindBestInterpolatedHit @@ -240,6 +245,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ N++; covYYUpd = mC[0]; } else if (retValHit == 1) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + } break; } @@ -413,7 +421,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } } -GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk) +GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk, bool interOnly) { const GPUParam& GPUrestrict() param = merger.Param(); const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); @@ -421,7 +429,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre GPUglobalref() const cahit2* hits = tracker.HitData(rowData); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(rowData); float uncorrectedY = -1e6f, uncorrectedZ; - if (rowData.NHits() && (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0 || (param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { + if (rowData.NHits() && (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0 || (!interOnly && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = rowData.Grid().YMin(); const float stepY = rowData.HstepY(); @@ -430,9 +438,14 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre int32_t bin, ny, nz; float ImP0, ImP1, ImC0, ImC2; - if (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - const float Iz0 = inter.posY - mP[0]; - const float Iz1 = inter.posZ + deltaZ - mP[1]; + if (interOnly) { + ImP0 = (float)inter.posY; + ImP1 = (float)inter.posZ + deltaZ; + ImC0 = (float)inter.errorY; + ImC2 = (float)inter.errorZ; + } else if (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + const float Iz0 = (float)inter.posY - mP[0]; + const float Iz1 = (float)inter.posZ + deltaZ - mP[1]; const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); const float Ik00 = mC[0] * Iw0; @@ -517,14 +530,33 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre } CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("\t\t\tiTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); } - if (nCandidates == 0) { + if (nCandidates == 0 && !interOnly) { // TODO: remove the interOnly here when rebuilding is fully working merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; } } return uncorrectedY; } -GPUdii() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, GPUTPCGMMerger& GPUrestrict() merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow) +GPUdni() void GPUTPCGMTrackParam::InterpolateMissingRows(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrors& GPUrestrict() interpolation, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk) +{ + for (; ihit >= 0; ihit--, interpolationIndex--) { + while (ihit > 0 && clusters[ihit].row == clusters[ihit - 1].row && clusters[ihit].sector == clusters[ihit - 1].sector) { + ihit--; + } + const auto& cluster = clusters[ihit]; + if (CAMath::Abs(cluster.row - lastRow) > 1) { + interpolationIndex -= CAMath::Abs(cluster.row - lastRow) - 1; + } + lastRow = cluster.row; + if (interpolationIndex < 0) { // TODO: Is this check needed? + return; + } + auto inter = interpolation.hit[interpolationIndex]; + FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); + } +} + +GPUd() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, GPUTPCGMMerger& GPUrestrict() merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow) { const GPUParam& GPUrestrict() param = merger.Param(); if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 1c5efc3b22ed7..9e3c0529eb675 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -163,7 +163,8 @@ class GPUTPCGMTrackParam GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); - GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); + GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, bool interOnly); + GPUd() void InterpolateMissingRows(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrors& interpolation, GPUTPCGMMergedTrackHit* clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); From d745b635a5b1a1ac10b993fe750f483b8b4d6ca1 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 31 Oct 2025 13:04:02 +0100 Subject: [PATCH 18/25] GPU TPC: Interpolate missing pad rows when rebuilding track --- GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h | 2 + GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 50 ++++++++++++------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 2 +- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h index c729ab7ef5b2e..e35c697c6ac49 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h @@ -34,6 +34,8 @@ enum attachTypes { attachProtect = 0x80000000, struct InterpolationErrorHit { float posY, posZ; GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A errorY, errorZ; + GPUdi() bool isValid() const { return errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0; } + GPUdi() void markInvalid() { errorY = (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)-1; } }; struct InterpolationErrors { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 275cf923c8d87..b37331f66991a 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -61,12 +61,11 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ prop.SetPolynomialField(¶m.polynomialField); prop.SetMaxSinPhi(maxSinPhi); if (param.rec.tpc.mergerInterpolateErrors && !rebuilt) { - for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing size - interpolation.hit[i].errorY = -1; + for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing range + interpolation.hit[i].markInvalid(); } if (param.rec.tpc.rebuildTrackInFit) { - merger.TrackRebuildHelper()[iTrk].highInclRowLow = 255; - merger.TrackRebuildHelper()[iTrk].highInclRowHigh = 255; + merger.TrackRebuildHelper()[iTrk].highInclRowLow = merger.TrackRebuildHelper()[iTrk].highInclRowHigh = 255; } } @@ -134,22 +133,24 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); uint8_t dEdxSubThresholdRow = 255; - bool doInterpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { - bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; - bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); - if (dodEdx || doAttach || doInterpolate) { + const bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; + const bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); + const uint8_t doInterpolate = (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 3) ? 1 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) ? 2 : 0); + if (lastUpdateRow != 255 && (dodEdx || doAttach || doInterpolate)) { int32_t step = cluster.row > lastPropagateRow ? 1 : -1; - for (int32_t iRow = lastPropagateRow + step; iRow != cluster.row; iRow += step) { + uint8_t sector = lastSector; + + for (int32_t iRow = lastPropagateRow + step, index = interpolationIndex; iRow != cluster.row; iRow += step, index += wayDirection) { float tmpX, tmpY, tmpZ; - if (prop.GetPropagatedYZ(mX - GPUTPCGeometry::Row2X(iRow - step) + GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { + if (prop.GetPropagatedYZ(GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { break; } - merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(cluster.sector, iRow, tmpY, tmpZ, tmpX); + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(sector, iRow, tmpY, tmpZ, tmpX); if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { break; } - FitAddRow(iRow, cluster.sector, iTrk, track, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate); + FitAddRow(iRow, sector, iTrk, track, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate, interpolation.hit[index], deltaZ, sumInvSqrtCharge, nAvgCharge); } } interpolationIndex += (CAMath::Abs(cluster.row - lastPropagateRow) - 1) * wayDirection; @@ -233,6 +234,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ merger.TrackRebuildHelper()[iTrk].highInclRowHigh = cluster.row; } } + // TODO: We can perhaps break here, if we pick up remaining rows } continue; } @@ -298,7 +300,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ return true; } -GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate) +GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge) { if (CAMath::Abs(mP[2]) > constants::MAX_SIN_PHI || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { return; @@ -317,6 +319,16 @@ GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t se if (doAttach) { AttachClusters(merger, sector, iRow, iTrk, track.Leg() == 0, prop); } + if (doInterpolate) { + if (doInterpolate == 2) { + if (inter.isValid() && CAMath::Abs(inter.posY - mP[0]) < 4.f) { + inter.markInvalid(); + } + FindBestInterpolatedHit(merger, inter, sector, iRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); + } else { + prop.InterpolateFill(&inter); + } + } } GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() param, const uint8_t sector, const uint8_t& lastSector) @@ -363,10 +375,10 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger if (param.rec.tpc.mergerInterpolateErrors) { if (iWay == nWays - 2) { if (!param.rec.tpc.rebuildTrackInFit) { - if (inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - rejectChi2 = true; - } else { + if (inter.isValid()) { retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + } else { + rejectChi2 = true; } } } else if (iWay == nWays - 1) { @@ -429,7 +441,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre GPUglobalref() const cahit2* hits = tracker.HitData(rowData); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(rowData); float uncorrectedY = -1e6f, uncorrectedZ; - if (rowData.NHits() && (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0 || (!interOnly && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { + if (rowData.NHits() && (inter.isValid() || (!interOnly && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = rowData.Grid().YMin(); const float stepY = rowData.HstepY(); @@ -443,7 +455,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre ImP1 = (float)inter.posZ + deltaZ; ImC0 = (float)inter.errorY; ImC2 = (float)inter.errorZ; - } else if (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + } else if (inter.isValid()) { const float Iz0 = (float)inter.posY - mP[0]; const float Iz1 = (float)inter.posZ + deltaZ - mP[1]; const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); @@ -520,7 +532,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre if (insert < param.rec.tpc.rebuildTrackInFitClusterCandidates) { for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; - } + } // TODO: Downweight non-interpolated candidates merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = row, .sector = sector, .error = err, .weight = 0, .best = 0}; nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 9e3c0529eb675..5bb90ad177725 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -157,7 +157,7 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); - GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); + GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); From 1262211361173b035c8ef59576dde51b795bbbd0 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 31 Oct 2025 14:07:16 +0100 Subject: [PATCH 19/25] Revert "GPU TPC: Remove option to retry refit if cluster rejection breaks the track, we will anyway rebuild the track in the future" This reverts commit 5c3d113f80687ecfcb3bf106b0fc7ad274fceda9. We can actually properly retry also for rebuilt tracks, we just run the retry on the last iteration after rebuilding. --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + .../Global/GPUChainTrackingMerger.cxx | 4 ++ GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 5 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 3 + GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 7 ++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 62 ++++++++++++------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 6 +- GPU/GPUTracking/qa/GPUQA.cxx | 2 +- 8 files changed, 60 insertions(+), 30 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index ca29b992a7284..b0a6f68e4bcd9 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -160,6 +160,7 @@ AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation inst AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF") AddOptionRTC(mergerNonInterpolateRejectMinNDF, uint8_t, 5, "", 0, "Minimum NDF of track for non-interpolated reject (both for chi2 and absolute distance)") AddOptionRTC(mergeCE, uint8_t, 1, "", 0, "Merge tracks accross the central electrode") +AddOptionRTC(retryRefit, int8_t, 1, "", 0, "Retry refit with seeding errors and without cluster rejection when fit fails") AddOptionRTC(enablePID, int8_t, 1, "", 0, "Enable PID response") AddOptionRTC(PID_useNsigma, int8_t, 1, "", 0, "Use nSigma instead of absolute distance in PID response") AddOptionRTC(adddEdxSubThresholdClusters, int8_t, 1, "", 0, "Add sub threshold clusters in TPC dEdx computation") diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index de56c4d21c079..81642e20c621a 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -248,6 +248,10 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRebuiltTracks, *mDebugFile); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); } + if (param().rec.tpc.retryRefit) { + runKernel(GetGridAuto(0), -1, param().rec.tpc.rebuildTrackInFit); + } + runKernel(GetGridAuto(0)); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRefit, *mDebugFile); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 3c25fde41866c..f8dc23ca7b491 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -402,13 +402,16 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) computePointerWithAlignment(mem, mTrackSort, mNMaxTracks); memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; - computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway + computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks does not save memory, other parts are larger anyway if (mRec->GetParam().rec.tpc.rebuildTrackInFit) { computePointerWithAlignment(mem, mSharedCount2, mNClusters); computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * mNMaxTracks * GPUTPCGeometry::NROWS * Param().rec.tpc.rebuildTrackInFitClusterCandidates); computePointerWithAlignment(mem, mTrackRebuildHelper, mNMaxTracks); computePointerWithAlignment(mem, mHitWeights, mNClusters); } + if (mRec->GetParam().rec.tpc.retryRefit) { + computePointerWithAlignment(mem, mRetryRefitIds, mNMaxTracks); + } memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLooperCandidates, mNMaxLooperMatches); // MergeLoopers 1-3 diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 3d6a2fcad3ad3..6875d74b01485 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -67,6 +67,7 @@ class GPUTPCGMMerger : public GPUProcessor static constexpr const int32_t NSECTORS = GPUTPCGeometry::NSECTORS; //* N sectors struct memory { + GPUAtomic(uint32_t) nRetryRefit; GPUAtomic(uint32_t) nLoopData; GPUAtomic(uint32_t) nUnpackedTracks; GPUAtomic(uint32_t) nMergedTracks; @@ -140,6 +141,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } GPUhdi() uint32_t* TrackOrderProcess() const { return mTrackOrderProcess; } + GPUhdi() uint32_t* RetryRefitIds() const { return mRetryRefitIds; } GPUhdi() uint8_t* ClusterStateExt() const { return mClusterStateExt; } GPUhdi() GPUTPCGMLoopData* LoopData() const { return mLoopData; } GPUhdi() memory* Memory() const { return mMemory; } @@ -322,6 +324,7 @@ class GPUTPCGMMerger : public GPUProcessor gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRangeMemory = nullptr; // memory for border tracks gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRange[NSECTORS]; // memory for border tracks memory* mMemory = nullptr; + uint32_t* mRetryRefitIds = nullptr; GPUTPCGMLoopData* mLoopData = nullptr; }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 083cdf76699a0..2ef0d0ba59748 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -21,9 +21,10 @@ using namespace o2::gpu; template <> GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode, int32_t rebuilt) { - GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, merger.NMergedTracks(), { - const int32_t i = mode ? merger.TrackOrderProcess()[ii] : ii; - GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, merger, rebuilt); + const int32_t iEnd = mode == -1 ? merger.Memory()->nRetryRefit : merger.NMergedTracks(); + GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, iEnd, { + const int32_t i = mode == -1 ? merger.RetryRefitIds()[ii] : (mode ? merger.TrackOrderProcess()[ii] : ii); + GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, merger, rebuilt, mode == -1); }); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index b37331f66991a..17f506334e64d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -47,7 +47,7 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt, bool retryAttempt) { static constexpr float maxSinPhi = constants::MAX_SIN_PHI; @@ -91,7 +91,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const bool finalFit = iWay == nWays - 1; ResetCovariance(); - prop.SetSeedingErrors(!(refit)); + prop.SetSeedingErrors(!(refit && !retryAttempt)); prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed prop.SetPropagateBzOnly(param.rec.fitPropagateBzOnly == -1 ? !finalFit : param.rec.fitPropagateBzOnly); prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger.GetConstantMem()->calibObjects.matLUT : nullptr); @@ -239,7 +239,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY, retryAttempt); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); ihitStart = ihit; @@ -256,7 +256,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ lastUpdateRow = cluster.row; assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } - if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255) { + if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255 && !retryAttempt) { StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } @@ -347,7 +347,7 @@ GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() pa } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY, bool retryAttempt) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -371,23 +371,25 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); - bool rejectChi2 = (clusterState & GPUTPCGMMergedTrackHit::flagReject); - if (param.rec.tpc.mergerInterpolateErrors) { - if (iWay == nWays - 2) { - if (!param.rec.tpc.rebuildTrackInFit) { - if (inter.isValid()) { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); - } else { + bool rejectChi2 = !retryAttempt && (clusterState & GPUTPCGMMergedTrackHit::flagReject); + if (!retryAttempt) { + if (param.rec.tpc.mergerInterpolateErrors) { + if (iWay == nWays - 2) { + if (!param.rec.tpc.rebuildTrackInFit) { + if (inter.isValid()) { + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + } else { + rejectChi2 = true; + } + } + } else if (iWay == nWays - 1) { + if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { rejectChi2 = true; } } - } else if (iWay == nWays - 1) { - if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { - rejectChi2 = true; - } + } else { + rejectChi2 = allowChangeClusters; } - } else { - rejectChi2 = allowChangeClusters; } if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { @@ -893,6 +895,10 @@ GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger& GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t loopIdx) { + GPUTPCGMLoopData& data = merger.LoopData()[loopIdx]; + if (!merger.MergedTracks()[data.track].OK()) { + return; + } GPUTPCGMPropagator prop; prop.SetMaterialTPC(); prop.SetPolynomialField(&merger.Param().polynomialField); @@ -902,7 +908,6 @@ GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestr prop.SetFitInProjections(true); prop.SetPropagateBzOnly(false); - GPUTPCGMLoopData& data = merger.LoopData()[loopIdx]; prop.SetTrack(&data.param, data.alpha); if (merger.Param().rec.tpc.looperFollowMode == 1) { data.param.AttachClustersLooperFollow(merger, prop, data.sector, data.track, data.outwards); @@ -1141,7 +1146,7 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const return ok; } -GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger& GPUrestrict() merger, bool rebuilt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. +GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger& GPUrestrict() merger, bool rebuilt, bool retryAttempt) { if (!track.OK()) { return; @@ -1153,16 +1158,29 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - bool ok = t.Fit(merger, iTrk, nTrackHits, NTolerated, Alpha, track, rebuilt); + bool ok = t.Fit(merger, iTrk, nTrackHits, NTolerated, Alpha, track, rebuilt, retryAttempt); CADEBUG(if (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); + if (!ok && (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) && !retryAttempt && merger.Param().rec.tpc.retryRefit) { + for (uint32_t i = 0; i < track.NClusters(); i++) { + merger.Clusters()[track.FirstClusterRef() + i].state &= GPUTPCGMMergedTrackHit::clustererAndSharedFlags; + } + CADEBUG(printf("Track rejected, marking for retry\n")); + uint32_t nRefit = CAMath::AtomicAdd(&merger.Memory()->nRetryRefit, 1u); + merger.RetryRefitIds()[nRefit] = iTrk; + return; + } + if (retryAttempt && (t.mNDF < 10 || t.mChi2 / t.mNDF >= 6.f)) { + ok = false; + } + if (CAMath::Abs(t.QPt()) < 1.e-4f) { t.QPt() = CAMath::Copysign(1.e-4f, t.QPt()); } CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger.MergedTracks()[iTrk].Looper()); }); - track.SetOK(ok); // TODO: Should we recover tracks who failed the fit in iWay0/1 for the rebuild? + track.SetOK(ok); if (t.GetNDF() <= 0 && !rebuilt && merger.Param().rec.tpc.rebuildTrackInFit) { // TODO: Better handling of NDF<0 tracks, how do we want to do cluster rejection? track.Param().NDF() = 0; } else { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 5bb90ad177725..f2e01674c3dd9 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -154,12 +154,12 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); + GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt, bool retryAttempt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY, bool retryAttempt); GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); - GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); + GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt, bool retryAttempt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); diff --git a/GPU/GPUTracking/qa/GPUQA.cxx b/GPU/GPUTracking/qa/GPUQA.cxx index d3266e8f7c5f1..d2debec1b63a7 100644 --- a/GPU/GPUTracking/qa/GPUQA.cxx +++ b/GPU/GPUTracking/qa/GPUQA.cxx @@ -2914,7 +2914,7 @@ int32_t GPUQA::DrawQAHistograms(TObjArray* qcout) } } - if (mQATasks & taskTrackStatistics) { + if (mQATasks & taskTrackStatistics) { // TODO: Add plot for Chi2/NDF, and NCl correct and fake MC histograms // Process track statistic histograms float tmpMax = 0.; for (int32_t k = 0; k < ConfigNumInputs; k++) { // TODO: Simplify this drawing, avoid copy&paste From 84049b1ba78b36c411307e586bcfc81dc17e9cd2 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 2 Nov 2025 11:06:13 +0100 Subject: [PATCH 20/25] GPU TPC: Relax chi2cut for single-pad clusters --- GPU/GPUTracking/Merger/GPUTPCGMPropagator.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h index e637c315f3ce6..3f1a74cfb6c26 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h @@ -108,13 +108,19 @@ class GPUTPCGMPropagator GPUd() float PredictChi2(float posY, float posZ, float err2Y, float err2Z) const; GPUd() static int32_t RejectCluster(float chiY, float chiZ, uint8_t clusterState) { - if (chiY > 9.f || chiZ > 9.f) { // TODO: Check how a track can have chi2/ncl > 18 + if (chiY > 9.f || chiZ > 9.f) { return 2; } - if ((chiY > 6.25f || chiZ > 6.25f) && (clusterState & (GPUTPCGMMergedTrackHit::flagSplit | GPUTPCGMMergedTrackHit::flagShared))) { + if (chiZ > 6.25f && (clusterState & (GPUTPCGMMergedTrackHit::flagSplit | GPUTPCGMMergedTrackHit::flagShared | GPUTPCGMMergedTrackHit::flagEdge | GPUTPCGMMergedTrackHit::flagSingle))) { return 2; } - if ((chiY > 1.f || chiZ > 6.25f) && (clusterState & (GPUTPCGMMergedTrackHit::flagEdge | GPUTPCGMMergedTrackHit::flagSingle))) { + if (chiY > 6.25f && (clusterState & (GPUTPCGMMergedTrackHit::flagSplit | GPUTPCGMMergedTrackHit::flagShared))) { + return 2; + } + if (chiY > 1.f && (clusterState & GPUTPCGMMergedTrackHit::flagEdge)) { + return 2; + } + if (chiY > 2.f && (clusterState & GPUTPCGMMergedTrackHit::flagSingle)) { return 2; } return 0; From 2be776466ab7881a8df8b26c381bf60d176c5182 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 2 Nov 2025 15:41:14 +0100 Subject: [PATCH 21/25] GPU TPC: Clean up and fix resetT0 code --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 19 ++++++++++--------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 13 +------------ 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 17f506334e64d..964dd14c5c587 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -85,7 +85,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ StoreOuter(&track.OuterParam(), prop.GetAlpha()); } - int32_t resetT0 = initResetT0(); + int32_t resetT0 = CAMath::Max(10, CAMath::Min(40, 150.f / CAMath::Abs(mP[4]))); + ; const bool refit = (nWays == 1 || iWay >= 1); const bool finalOutInFit = iWay + 2 >= nWays; const bool finalFit = iWay == nWays - 1; @@ -239,9 +240,15 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY, retryAttempt); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, uncorrectedY, retryAttempt); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); + if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && CAMath::Square(mP[0] - prop.Model().Y()) + CAMath::Square(mP[1] - prop.Model().Z()) > 1) { + CADEBUG(printf("Reinit linearization\n")); + prop.SetTrack(this, prop.GetAlpha()); + resetT0 = 20; + } + ihitStart = ihit; interpolatedStart = interpolationIndex; N++; @@ -347,7 +354,7 @@ GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() pa } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY, bool retryAttempt) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, float uncorrectedY, bool retryAttempt) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -412,12 +419,6 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger lastUpdateX = mX; nMissed = nMissed2 = 0; UnmarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - float dy = mP[0] - prop.Model().Y(); - float dz = mP[1] - prop.Model().Z(); - if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && dy * dy + dz * dz > 1) { - CADEBUG(printf("Reinit linearization\n")); - prop.SetTrack(this, prop.GetAlpha()); - } return 0; // ok } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track if (retValInt || allowChangeClusters) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index f2e01674c3dd9..e083933344648 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -156,7 +156,7 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt, bool retryAttempt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY, bool retryAttempt); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float uncorrectedY, bool retryAttempt); GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt, bool retryAttempt); @@ -229,8 +229,6 @@ class GPUTPCGMTrackParam } private: - GPUd() int32_t initResetT0(); - float mX; // x position float mTOffset; // Z offset with early transform, T offset otherwise float mP[5]; // 'active' track parameters: Y, Z, SinPhi, DzDs, q/Pt @@ -248,15 +246,6 @@ struct GPUTPCGMLoopData { uint8_t outwards; }; -GPUdi() int32_t GPUTPCGMTrackParam::initResetT0() -{ - const float absQPt = CAMath::Abs(mP[4]); - if (absQPt < (150.f / 40.f)) { - return 150.f / 40.f; - } - return CAMath::Max(10.f, 150.f / mP[4]); -} - GPUdi() void GPUTPCGMTrackParam::ResetCovariance() { mC[0] = 100.f; From 5704a97077f18d320801f159fff21d188902a04b Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 2 Nov 2025 15:42:27 +0100 Subject: [PATCH 22/25] GPU TPC: Don't break for high sinPhi if only the linearization model is out of range, but instead reinitialize the model --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 964dd14c5c587..fb81f007c7b78 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -218,26 +218,31 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); - if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - nMissed2++; - CADEBUG(printf(" --- break-sinphi\n")); - NTolerated++; + if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { // TODO: If NDF is large enough, and mP[2] is not yet out of range, reinit linearization const bool inward = clusters[0].row > clusters[maxN - 1].row; const bool markHighIncl = (mP[2] > 0) ^ (mP[4] < 0) ^ inward ^ (iWay & 1); - if (param.rec.tpc.rebuildTrackInFit && markHighIncl) { - if (inward ^ (iWay & 1)) { - if (merger.TrackRebuildHelper()[iTrk].highInclRowLow == 255) { - merger.TrackRebuildHelper()[iTrk].highInclRowLow = cluster.row; - } - } else { - if (merger.TrackRebuildHelper()[iTrk].highInclRowHigh == 255) { - merger.TrackRebuildHelper()[iTrk].highInclRowHigh = cluster.row; + if (mNDF > 10 && CAMath::Abs(mP[2]) < maxSinForUpdate && markHighIncl) { + CADEBUG(printf("Reinit linearization\n")); + prop.SetTrack(this, prop.GetAlpha()); + } else { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); + nMissed2++; + CADEBUG(printf(" --- break-sinphi\n")); + NTolerated++; + if (param.rec.tpc.rebuildTrackInFit && markHighIncl) { + if (inward ^ (iWay & 1)) { + if (merger.TrackRebuildHelper()[iTrk].highInclRowLow == 255) { + merger.TrackRebuildHelper()[iTrk].highInclRowLow = cluster.row; + } + } else { + if (merger.TrackRebuildHelper()[iTrk].highInclRowHigh == 255) { + merger.TrackRebuildHelper()[iTrk].highInclRowHigh = cluster.row; + } } + // TODO: We can perhaps break here, if we pick up remaining rows } - // TODO: We can perhaps break here, if we pick up remaining rows + continue; } - continue; } int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, uncorrectedY, retryAttempt); From a2439dea5bc68b3952a101afb8f7e268b17157b4 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 18 Nov 2025 15:37:23 +0100 Subject: [PATCH 23/25] GPU: Add option to steer which attachment steps to run during rebuild --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 7 +++--- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 22 ++++++++++--------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 21 +++++++++--------- .../Standalone/Benchmark/standalone.cxx | 2 +- 4 files changed, 28 insertions(+), 24 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index b0a6f68e4bcd9..fdf6dfc255317 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -142,13 +142,14 @@ AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") +AddOptionRTC(disableRebuildAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during track rebuid (1: current row, 2: interpolate missing row, 4: one sided interpolation, 8: original hit)") +AddOptionRTC(disableMarkAdjacent, uint8_t, 0, "", 0, "Bitmask to disable certain steps during refit to mark adjacenrt clusters (1: current row, 2: extrapolate missing rows, 4: loop following)") // TODO: Add history AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking (prolong tracks to adjacent sectors to find short segments)") -AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit (1: attachment, 2: propagation, 4: loop following)") -AddOptionRTC(looperFollowMode, uint8_t, 1, "", 0, "0 = simple, 1 = advances, for disabling use disableRefitAttachment = 4") +AddOptionRTC(looperFollowMode, uint8_t, 1, "", 0, "0 = simple, 1 = advances, for disabling use disableMarkAdjacent = 4") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression (0 = no, 1 = strategy A, 2 = strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits (1=truncate charge/width LSB, 2=differences, 4=track-model)") @@ -171,7 +172,7 @@ AddOptionRTC(rejectEdgeClustersInTrackFit, int8_t, 0, "", 0, "Reject edge cluste AddOptionRTC(tubeExtraProtectMinRow, uint8_t, 20, "", 0, "Increase Protection, decrease removal by factor 2, when below this row") AddOptionRTC(tubeExtraProtectEdgePads, uint8_t, 2, "", 0, "Increase Protection, decrease removal by factor 2, when on this number of pads from the edge") -AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet hand AddOptionArrayRTC +AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet handle AddOptionArrayRTC AddHelp("help", 'h') EndConfig() diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index f8dc23ca7b491..cd49668ca7512 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -2144,16 +2144,18 @@ GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, continue; } mTrackRebuildHelper[i].reverse = mClusters[trk.FirstClusterRef()].row < mClusters[trk.FirstClusterRef() + trk.NClusters() - 1].row; - int lastRow = -1; - for (uint32_t j = 0; j < trk.NClusters(); j++) { - const auto& cl = mClusters[trk.FirstClusterRef() + j]; - auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (cl.row != lastRow && candidates[0].id == 0 && (!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0)) { - candidates[0].id = cl.num + 2; - candidates[0].best = 128; - candidates[0].weight = cl.state; - candidates[0].sector = cl.sector; - lastRow = cl.row; + if (!(Param().rec.tpc.disableRebuildAttachment & 8)) { + int lastRow = -1; + for (uint32_t j = 0; j < trk.NClusters(); j++) { + const auto& cl = mClusters[trk.FirstClusterRef() + j]; + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (cl.row != lastRow && candidates[0].id == 0 && (!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0)) { + candidates[0].id = cl.num + 2; + candidates[0].best = 128; + candidates[0].weight = cl.state; + candidates[0].sector = cl.sector; + lastRow = cl.row; + } } } } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index fb81f007c7b78..5079f5b2bc0e0 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -136,8 +136,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t dEdxSubThresholdRow = 255; if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { const bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; - const bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); - const uint8_t doInterpolate = (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 3) ? 1 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) ? 2 : 0); + const bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableMarkAdjacent & 2); + const uint8_t doInterpolate = (param.rec.tpc.disableRebuildAttachment & 2) ? 0 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 3) ? 1 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) ? 2 : 0)); if (lastUpdateRow != 255 && (dodEdx || doAttach || doInterpolate)) { int32_t step = cluster.row > lastPropagateRow ? 1 : -1; uint8_t sector = lastSector; @@ -168,7 +168,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; NTolerated++; - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); // TODO: mark clusters from single-sided finding as dubious, and apply stricter restriction cut later } continue; @@ -178,7 +178,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); // clang-format on if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); } break; // bad chi2 for the whole track, stop the fit @@ -191,15 +191,16 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit) { - if (iWay == nWays - 2) { - uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); + if (iWay == nWays - 2 && !(param.rec.tpc.disableRebuildAttachment & 1)) { + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); // TODO: Check whether we do not need to use track Y for some rejection steps } if (allowChangeClusters) { AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); // TODO: Do this during FindBestInterpolatedHit } } else if (allowChangeClusters) { uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); - } else if (param.rec.tpc.rejectEdgeClustersInTrackFit) { + } + if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY <= -1e6f) { float tmpZ; merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, mP[0], mP[1], uncorrectedY, tmpZ); } @@ -259,7 +260,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ N++; covYYUpd = mC[0]; } else if (retValHit == 1) { - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); } break; @@ -268,7 +269,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ lastUpdateRow = cluster.row; assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } - if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255 && !retryAttempt) { + if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } @@ -772,7 +773,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestric GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z) { const GPUParam& GPUrestrict() param = merger.Param(); - if (param.rec.tpc.disableRefitAttachment & 1) { + if (param.rec.tpc.disableMarkAdjacent & 1) { return -1e6f; } const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); diff --git a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx index 85a2b718c0595..80421855913e1 100644 --- a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx +++ b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx @@ -498,7 +498,7 @@ int32_t SetupReconstruction() if (recSet.tpc.rejectionStrategy >= GPUSettings::RejectionStrategyB) { procSet.tpcInputWithClusterRejection = 1; } - recSet.tpc.disableRefitAttachment = 0xFF; + recSet.tpc.disableMarkAdjacent = 0xFF; recSet.maxTrackQPtB5 = CAMath::Min(recSet.maxTrackQPtB5, recSet.tpc.rejectQPtB5); GPUChainTracking::ApplySyncSettings(procSet, recSet, steps.steps, false, configStandalone.rundEdx); recAsync->SetSettings(&grp, &recSet, &procSet, &steps); From ab7a0c43d604216a4a323b75a2f0e5729efd13f3 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 2 Nov 2025 19:19:38 +0100 Subject: [PATCH 24/25] GPU TPC: Extrapolate track inward and outward when rebuilding --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 6 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 114 +++++++++++++++++- 2 files changed, 113 insertions(+), 7 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index fdf6dfc255317..255023d788650 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -142,7 +142,7 @@ AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") -AddOptionRTC(disableRebuildAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during track rebuid (1: current row, 2: interpolate missing row, 4: one sided interpolation, 8: original hit)") +AddOptionRTC(disableRebuildAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during track rebuid (1: current row, 2: interpolate missing row, 4: one sided interpolation, 8: original hit, 16: extrapolation)") AddOptionRTC(disableMarkAdjacent, uint8_t, 0, "", 0, "Bitmask to disable certain steps during refit to mark adjacenrt clusters (1: current row, 2: extrapolate missing rows, 4: loop following)") // TODO: Add history AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") @@ -171,7 +171,9 @@ AddOptionRTC(rejectEdgeClustersInSeeding, int8_t, 0, "", 0, "Reject edge cluster AddOptionRTC(rejectEdgeClustersInTrackFit, int8_t, 0, "", 0, "Reject edge clusters based on uncorrected track Y during track fit") AddOptionRTC(tubeExtraProtectMinRow, uint8_t, 20, "", 0, "Increase Protection, decrease removal by factor 2, when below this row") AddOptionRTC(tubeExtraProtectEdgePads, uint8_t, 2, "", 0, "Increase Protection, decrease removal by factor 2, when on this number of pads from the edge") - +AddOptionRTC(rebuildTrackExtrMaxMissingRows, uint8_t, 15, "", 0, "Maximum total number of rows allowed to be missing during track extrapolation") +AddOptionRTC(rebuildTrackExtrMinConsecGoodRows, uint8_t, 6, "", 0, "Minimum number of consecutive rows required to accept extrapolated segment") +AddOptionRTC(rebuildTrackExtrConsecGoodRowsMaxGap, uint8_t, 1, "", 0, "Max row gap size to consider the segment as consecutive rows") AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet handle AddOptionArrayRTC AddHelp("help", 'h') EndConfig() diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 5079f5b2bc0e0..d07142ff7b802 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -50,6 +50,7 @@ using namespace o2::tpc; GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt, bool retryAttempt) { static constexpr float maxSinPhi = constants::MAX_SIN_PHI; + const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); const GPUParam& GPUrestrict() param = merger.Param(); GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); @@ -218,7 +219,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } - const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { // TODO: If NDF is large enough, and mP[2] is not yet out of range, reinit linearization const bool inward = clusters[0].row > clusters[maxN - 1].row; const bool markHighIncl = (mP[2] > 0) ^ (mP[4] < 0) ^ inward ^ (iWay & 1); @@ -269,16 +269,120 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ lastUpdateRow = cluster.row; assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } - if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { - StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); - CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); - } if (!(iWay & 1) && !finalFit && !track.CCE() && !track.Looper()) { deltaZ = ShiftZ(clusters, merger, maxN); } else { deltaZ = 0.f; } + if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && iWay >= nWays - 3 && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255) { + const int32_t up = ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1)) ? 1 : -1; + int32_t sector = lastSector; + uint8_t rowGapActive = 0, rowGapTotal = 0, missingRowsTotal = 0; + uint8_t lastGoodRow = lastPropagateRow, lastExtrapolateRow = lastPropagateRow; + uint8_t consecGoodRows = param.rec.tpc.rebuildTrackExtrMinConsecGoodRows, consecGoodRowsMissing = 0; + prop.SetTrack(this, prop.GetAlpha()); + for (uint32_t iRow = lastPropagateRow + up; iRow < GPUTPCGeometry::NROWS; iRow += up) { // uint8_t implies > 0! // TODO: Try to reduce some variables to int8/uint8 to save registers + bool fail = false; + for (int32_t iAttempt = 0; iAttempt < 2; iAttempt++) { + float tmpX, tmpY, tmpZ; + if (prop.GetPropagatedYZ(GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { + fail = true; + break; + } + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(sector, iRow, tmpY, tmpZ, tmpX); + if (prop.PropagateToXAlpha(tmpX, param.Alpha(sector), inFlyDirection)) { + fail = true; + break; + } + if (CAMath::Abs(mP[2]) > maxSinForUpdate) { + fail = true; + break; + } + if (CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f) + 0.1f) { + if (iAttempt) { + fail = true; + break; + } + const int32_t sectorSide = sector >= (int8_t)(GPUTPCGeometry::NSECTORS / 2) ? (GPUTPCGeometry::NSECTORS / 2) : 0; + if (mP[0] > 0) { + if (++sector >= sectorSide + 18) { + sector -= 18; + } + } else { + if (--sector < sectorSide) { + sector += 18; + } + } + } + } + if (fail) { + break; + } + CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Propaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", sector, iRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); + gputpcgmmergertypes::InterpolationErrorHit inter; + inter.markInvalid(); + float uncorrectedY = FindBestInterpolatedHit(merger, inter, sector, iRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); + auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; + if (candidate.id >= 2) { + lastExtrapolateRow = iRow; + float err2Y, err2Z, xx, yy, zz; + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[candidate.id - 2]; + merger.GetConstantMem()->calibObjects.fastTransform->Transform(sector, iRow, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); + if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection)) { + candidate.best = -1; + break; + } + const float time = merger.GetConstantMem()->ioPtrs.clustersNative ? cl.getTime() : -1.f; // TODO: When is it possible that we do not have clusterNative? + const float invSqrtCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? CAMath::InvSqrt(cl.qMax) : 0.f; + const float invCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? (1.f / cl.qMax) : 0.f; + float invAvgCharge = (sumInvSqrtCharge += invSqrtCharge) / ++nAvgCharge; + invAvgCharge *= invAvgCharge; + const uint8_t clusterState = cl.getFlags(); + prop.GetErr2(err2Y, err2Z, param, zz, iRow, clusterState, sector, time, invAvgCharge, invCharge); + CADEBUG(printf("\t%21sResiduals %8.3f %8.3f --- Errors %8.3f %8.3f\n", "", yy - mP[0], zz - mP[1], CAMath::Sqrt(err2Y), CAMath::Sqrt(err2Z))); + uint32_t retValUpd = prop.Update(yy, zz, iRow, param, clusterState, true, refit, err2Y, err2Z); + CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Fit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d\n", sector, iRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd)); + if (retValUpd == 1) { + candidate.best = -1; + break; + } + if (++consecGoodRows >= param.rec.tpc.rebuildTrackExtrMinConsecGoodRows) { + lastGoodRow = iRow; + consecGoodRowsMissing = 0; + } + rowGapActive = rowGapTotal = 0; + } else { + if (++missingRowsTotal > param.rec.tpc.rebuildTrackExtrMaxMissingRows) { + break; + } + if (++rowGapTotal > param.rec.tpc.trackFollowingMaxRowGap) { + consecGoodRows = consecGoodRowsMissing = 0; + } + uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, uncorrectedY)); + if (pad < GPUTPCGeometry::NPads(iRow) && (!merger.GetConstantMem()->calibObjects.dEdxCalibContainer || !merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad))) { + if (rowGapActive++ >= param.rec.tpc.trackFollowingMaxRowGap) { + break; + } + if (consecGoodRowsMissing++ >= param.rec.tpc.rebuildTrackExtrConsecGoodRowsMaxGap) { + consecGoodRows = consecGoodRowsMissing = 0; + } + } + } + } + if (lastGoodRow != lastExtrapolateRow) { + for (int32_t iRow = lastGoodRow + up; iRow != lastExtrapolateRow + up; iRow += up) { + auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; + candidate.best = -1; + } + } + } + + if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { + StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); + CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); + } + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { Alpha = prop.GetAlpha(); if (ihitStart != 0) { From a5bf9d0fce6ca62df5224905db9f2faee9838a23 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 4 Dec 2025 12:05:46 +0100 Subject: [PATCH 25/25] GPU: Add debug option to create temporary MC labels for collected merged tracks --- GPU/GPUTracking/Base/GPUReconstructionCPU.cxx | 4 +-- GPU/GPUTracking/Definitions/GPUSettingsList.h | 13 +++++-- .../GPUTrackingLinkDef_O2_DataTypes.h | 1 + .../Global/GPUChainTrackingMerger.cxx | 3 ++ GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 34 ++++++++++++++++++- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 3 ++ GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 13 +++++++ GPU/GPUTracking/qa/GPUQAHelper.h | 11 ++++++ 8 files changed, 76 insertions(+), 6 deletions(-) diff --git a/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx b/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx index 9fbe9e1171af3..50324d3bfc99f 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx +++ b/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx @@ -216,10 +216,10 @@ int32_t GPUReconstructionCPU::ExitDevice() int32_t GPUReconstructionCPU::RunChains() { mMemoryScalers->temporaryFactor = 1.; - if (GetProcessingSettings().memoryScalingFuzz) { + if (GetProcessingSettings().debug.memoryScalingFuzz) { static std::mt19937 rng; static std::uniform_int_distribution dist(0, 1000000); - uint64_t fuzzFactor = GetProcessingSettings().memoryScalingFuzz == 1 ? dist(rng) : GetProcessingSettings().memoryScalingFuzz; + uint64_t fuzzFactor = GetProcessingSettings().debug.memoryScalingFuzz == 1 ? dist(rng) : GetProcessingSettings().debug.memoryScalingFuzz; GPUInfo("Fuzzing memory scaling factor with %lu", fuzzFactor); mMemoryScalers->fuzzScalingFactor(fuzzFactor); } diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 255023d788650..38fda6a3cc5f5 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -39,7 +39,7 @@ BeginNamespace(gpu) // Reconstruction parameters for TPC, no bool in here !!! BeginSubConfig(GPUSettingsRecTPC, tpc, configStandalone.rec, "RECTPC", 0, "Reconstruction settings", rec_tpc) -AddOptionRTC(rejectQPtB5, float, 1.f / 0.050f, "", 0, "QPt threshold to reject clusters of TPC tracks (Inverse Pt, scaled to B=0.5T!!!)") +AddOptionRTC(rejectQPtB5, float, 1.f / 0.050f, "", 0, "QPt threshold to reject clusters of TPC tracks (Inverse Pt, scaled to B=0.5T!!!)") // TODO: Sort these options automatically for parameter size AddOptionRTC(hitPickUpFactor, float, 1.f, "", 0, "multiplier for the combined cluster+track error during track following") AddOptionRTC(hitSearchArea2, float, 2.f, "", 0, "square of maximum search road of hits during seeding") AddOptionRTC(neighboursSearchArea, float, 3.f, "", 0, "area in cm for the search of neighbours, for z only used if searchWindowDZDR = 0") @@ -326,6 +326,13 @@ AddOption(conservativeMemoryEstimate, bool, false, "", 0, "Use some more conserv AddHelp("help", 'h') EndConfig() +// Debug Settings +BeginSubConfig(GPUSettingsProcessingDebug, debug, configStandalone.proc, "DEBUG", 0, "Debugging Settings", proc_debug) +AddOption(memoryScalingFuzz, uint64_t, 0, "", 0, "Fuzz the memoryScalingFactor (0 disable, 1 enable, >1 set seed", def(1)) +AddOption(mergerMCLabels, bool, false, "", 0, "Create MC labels for merged tracks before refit for debugging") +AddHelp("help", 'h') +EndConfig() + // Settings steering the processing once the device was selected, only available on the host BeginSubConfig(GPUSettingsProcessing, proc, configStandalone, "PROC", 0, "Processing settings", proc) AddOption(deviceNum, int32_t, -1, "gpuDevice", 0, "Set GPU device to use (-1: automatic, -2: for round-robin usage in timeslice-pipeline)") @@ -354,7 +361,6 @@ AddOption(memoryAllocationStrategy, int8_t, 0, "", 0, "Memory Allocation Strageg AddOption(forceMemoryPoolSize, uint64_t, 1, "memSize", 0, "Force size of allocated GPU / page locked host memory", min(0ul)) AddOption(forceHostMemoryPoolSize, uint64_t, 0, "hostMemSize", 0, "Force size of allocated host page locked host memory (overriding memSize)", min(0ul)) AddOption(memoryScalingFactor, float, 1.f, "", 0, "Factor to apply to all memory scalers") -AddOption(memoryScalingFuzz, uint64_t, 0, "", 0, "Fuzz the memoryScalingFactor (0 disable, 1 enable, >1 set seed", def(1)) AddOption(tpcInputWithClusterRejection, uint8_t, 0, "", 0, "Indicate whether the TPC input is CTF data with cluster rejection, to tune buffer estimations") AddOption(forceMaxMemScalers, uint64_t, 0, "", 0, "Force using the maximum values for all buffers, Set a value n > 1 to rescale all maximums to a memory size of n") AddOption(registerStandaloneInputMemory, bool, false, "registerInputMemory", 0, "Automatically register input memory buffers for the GPU") @@ -401,7 +407,7 @@ AddOption(tpcUseOldCPUDecoding, bool, false, "", 0, "Enable old CPU-based TPC de AddOption(tpcApplyCFCutsAtDecoding, bool, false, "", 0, "Apply cluster cuts from clusterization during decoding of compressed clusters") AddOption(tpcApplyClusterFilterOnCPU, uint8_t, 0, "", 0, "Apply custom cluster filter of GPUTPCClusterFilter class, 0: off, 1: debug, 2: PbPb23") AddOption(tpcWriteClustersAfterRejection, bool, false, "", 0, "Apply TPC rejection strategy before writing clusters") -AddOption(oclPlatformNum, int32_t, -1, "", 0, "Platform to use, in case the backend provides multiple platforms (OpenCL only, -1 = auto-select, -2 query all platforms (also incompatible))") +AddOption(oclPlatformNum, int32_t, -1, "", 0, "Platform to use, in case the backend provides multiple platforms (OpenCL only, -1 = auto-select, -2 query all platforms (also incompatible))") // TODO: Create some backend-specific options AddOption(oclCompileFromSources, bool, false, "", 0, "Compile OpenCL binary from included source code instead of using included spirv code") AddOption(oclOverrideSourceBuildFlags, std::string, "", "", 0, "Override OCL build flags for compilation from source, put a space for empty options") AddOption(hipOverrideAMDEUSperCU, int32_t, -1, "", 0, "Override AMD_EUS_PER_CU setting") @@ -422,6 +428,7 @@ AddSubConfig(GPUSettingsProcessingRTCtechnical, rtctech) AddSubConfig(GPUSettingsProcessingParam, param) AddSubConfig(GPUSettingsProcessingNNclusterizer, nn) AddSubConfig(GPUSettingsProcessingScaling, scaling) +AddSubConfig(GPUSettingsProcessingDebug, debug) AddHelp("help", 'h') EndConfig() #endif // __OPENCL__ diff --git a/GPU/GPUTracking/GPUTrackingLinkDef_O2_DataTypes.h b/GPU/GPUTracking/GPUTrackingLinkDef_O2_DataTypes.h index 98be3dd72baba..3f14a29ceb63f 100644 --- a/GPU/GPUTracking/GPUTrackingLinkDef_O2_DataTypes.h +++ b/GPU/GPUTracking/GPUTrackingLinkDef_O2_DataTypes.h @@ -33,6 +33,7 @@ #pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsProcessingRTCtechnical + ; #pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsProcessingNNclusterizer + ; #pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsProcessingScaling + ; +#pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsProcessingDebug + ; #pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsDisplay + ; #pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsDisplayLight + ; #pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsDisplayHeavy + ; diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 81642e20c621a..022d5da8a1b80 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -176,6 +176,9 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) runKernel({{1, -WarpSize(), 0, deviceType}}, 1); runKernel({{1, -WarpSize(), 0, deviceType}}, 1); } + if (!doGPU && GetProcessingSettings().debug.mergerMCLabels) { + Merger.CreateMCLabels(1, 1, 0, 0); + } DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingCollectedTracks, doGPU, Merger, &GPUTPCGMMerger::DumpCollected, *mDebugFile); if (param().rec.tpc.mergeCE) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index cd49668ca7512..14531c81d034e 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -148,6 +148,7 @@ using namespace o2::gpu::internal; #include "GPUQA.h" #include "GPUMemorySizeScalers.h" +#include "GPUQAHelper.h" GPUTPCGMMerger::GPUTPCGMMerger() { @@ -164,7 +165,7 @@ GPUTPCGMMerger::GPUTPCGMMerger() } // DEBUG CODE -#if !defined(GPUCA_GPUCODE) && (defined(GPUCA_MERGER_BY_MC_LABEL) || defined(GPUCA_CADEBUG_ENABLED) || GPUCA_MERGE_LOOPER_MC) +#if defined(GPUCA_MERGER_BY_MC_LABEL) || defined(GPUCA_CADEBUG_ENABLED) || GPUCA_MERGE_LOOPER_MC #include "GPUQAHelper.h" template @@ -438,6 +439,9 @@ void* GPUTPCGMMerger::SetPointersRefitScratch(void* mem) void* GPUTPCGMMerger::SetPointersOutput(void* mem) { computePointerWithAlignment(mem, mMergedTracks, mNMaxTracks); + if (mRec->GetProcessingSettings().debug.mergerMCLabels) { + computePointerWithAlignment(mem, mMergedTrackMC, mNMaxTracks); + } if (mRec->GetParam().dodEdxEnabled) { computePointerWithAlignment(mem, mMergedTracksdEdx, mNMaxTracks); if (mRec->GetParam().rec.tpc.dEdxClusterRejectionFlagMask != mRec->GetParam().rec.tpc.dEdxClusterRejectionFlagMaskAlt) { @@ -547,6 +551,34 @@ int32_t GPUTPCGMMerger::CheckSectors() return 0; } +void GPUTPCGMMerger::CreateMCLabels(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) +{ + const o2::tpc::ClusterNativeAccess* GPUrestrict() clusters = GetConstantMem()->ioPtrs.clustersNative; + if (clusters == nullptr || clusters->clustersMCTruth == nullptr) { + return; + } + if (mMergedTrackMC == nullptr) { + return; + } + + auto labelAssigner = GPUTPCTrkLbl(clusters->clustersMCTruth, 0.1f); + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + const auto& trk = mMergedTracks[i]; + if (!trk.OK()) { + continue; + } + labelAssigner.reset(); + for (uint32_t j = 0; j < trk.NClusters(); j++) { + const auto& cl = mClusters[trk.FirstClusterRef() + j]; + if (cl.state & GPUTPCGMMergedTrackHit::flagReject) { + continue; + } + labelAssigner.addLabel(cl.num); + } + mMergedTrackMC[i] = labelAssigner.computeLabel(); + } +} + #endif // GPUCA_GPUCODE GPUd() void GPUTPCGMMerger::ClearTrackLinks(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, bool output) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 6875d74b01485..032bc0a15e3fd 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -123,6 +123,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() int32_t NMergedTracks() const { return mMemory->nMergedTracks; } GPUhdi() const GPUTPCGMMergedTrack* MergedTracks() const { return mMergedTracks; } + GPUhdi() const o2::MCCompLabel* MergedTrackMC() const { return mMergedTrackMC; } GPUhdi() GPUTPCGMMergedTrack* MergedTracks() { return mMergedTracks; } GPUhdi() const GPUdEdxInfo* MergedTracksdEdx() const { return mMergedTracksdEdx; } GPUhdi() GPUdEdxInfo* MergedTracksdEdx() { return mMergedTracksdEdx; } @@ -214,6 +215,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUd() void ResolveHitWeights1(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration); GPUd() void ResolveHitWeights2(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void ResolveHitWeightsShared(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() void CreateMCLabels(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); #ifndef GPUCA_GPUCODE void DumpSectorTracks(std::ostream& out) const; @@ -294,6 +296,7 @@ class GPUTPCGMMerger : public GPUProcessor int32_t mNSectorHits = 0; // Total number of incoming clusters (from sector tracks) GPUTPCGMMergedTrack* mMergedTracks = nullptr; //* array of output merged tracks + o2::MCCompLabel* mMergedTrackMC = nullptr; trackCluster* mClusterCandidates = nullptr; trackRebuildHelper* mTrackRebuildHelper = nullptr; int32_t* mHitWeights = nullptr; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index d07142ff7b802..913873b29e72d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -14,6 +14,7 @@ #define GPUCA_CADEBUG 0 #define DEBUG_SINGLE_TRACK -1 +// #define DEBUG_REBUILD_MC #include "GPUTPCDef.h" #include "GPUTPCGMTrackParam.h" @@ -39,6 +40,11 @@ #include "AliHLTTPCClusterMCData.h" #endif +#ifndef GPUCA_GPUCODE +#include "SimulationDataFormat/ConstMCTruthContainer.h" +#include "SimulationDataFormat/MCCompLabel.h" +#endif + #ifndef GPUCA_GPUCODE_DEVICE #include #include @@ -278,6 +284,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && iWay >= nWays - 3 && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255) { const int32_t up = ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1)) ? 1 : -1; int32_t sector = lastSector; + CADEBUG(merger.MergedTrackMC() printf("Extrapolate Start Track %d - sector %2d row %3d %s - fake %d\n", iTrk, sector, (int32_t)lastPropagateRow, up == 1 ? "upwards" : "downwards", (int)merger.MergedTrackMC()[iTrk].isFake())); uint8_t rowGapActive = 0, rowGapTotal = 0, missingRowsTotal = 0; uint8_t lastGoodRow = lastPropagateRow, lastExtrapolateRow = lastPropagateRow; uint8_t consecGoodRows = param.rec.tpc.rebuildTrackExtrMinConsecGoodRows, consecGoodRowsMissing = 0; @@ -326,6 +333,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; if (candidate.id >= 2) { lastExtrapolateRow = iRow; +#if defined(DEBUG_REBUILD_MC) && !defined(GPUCA_GPUCODE) + if (merger.MergedTrackMC() && merger.GetConstantMem()->ioPtrs.clustersNative->clustersMCTruth) { + int32_t labelCorrect = GPUTPCTrkLblSearch(merger.GetConstantMem()->ioPtrs.clustersNative->clustersMCTruth->getLabels(candidate.id - 2), merger.MergedTrackMC()[iTrk]); + CADEBUG(printf("\t%21sLabel correct: %d\n", "", labelCorrect)); + } +#endif float err2Y, err2Z, xx, yy, zz; const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[candidate.id - 2]; merger.GetConstantMem()->calibObjects.fastTransform->Transform(sector, iRow, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); diff --git a/GPU/GPUTracking/qa/GPUQAHelper.h b/GPU/GPUTracking/qa/GPUQAHelper.h index dcbe7a94711aa..6712fd0a603ed 100644 --- a/GPU/GPUTracking/qa/GPUQAHelper.h +++ b/GPU/GPUTracking/qa/GPUQAHelper.h @@ -162,6 +162,17 @@ static inline auto GPUTPCTrkLbl(const AliHLTTPCClusterMCLabel* x, Args... args) } } +template +static inline bool GPUTPCTrkLblSearch(const T& clusterLabels, const MCCompLabel& trkLabel) +{ + for (const auto& clLabel : clusterLabels) { + if (trkLabel.compare(clLabel) >= 0) { + return true; + } + } + return false; +} + } // namespace gpu } // namespace o2