Skip to content

Commit 9257bc9

Browse files
cima22cima22
committed
ITS: new CPU + GPU seeding vertexer
Adds a seeding vertexer that runs as a prepended tracker pass (diamond trackleting -> cells -> lines -> parallel seeding), on both the CPU and GPU traits, replacing the per-ROF CPU vertexer for the seeding step.
1 parent 1d64083 commit 9257bc9

29 files changed

Lines changed: 2707 additions & 333 deletions

‎DataFormats/Detectors/ITSMFT/ITS/include/DataFormatsITS/Vertex.h‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
#include "GPUCommonDef.h"
1616
#ifndef GPUCA_GPUCODE_DEVICE
1717
#include <type_traits>
18+
#include <unordered_map>
19+
#include <utility>
1820
#endif
1921
#include "ReconstructionDataFormats/Vertex.h"
2022
#include "SimulationDataFormat/MCCompLabel.h"
@@ -25,6 +27,36 @@ namespace o2::its
2527
// NOTE: this uses the internal asymmetrical time reprenstation!
2628
using Vertex = o2::dataformats::Vertex<o2::its::TimeEstBC>;
2729
using VertexLabel = std::pair<o2::MCCompLabel, float>;
30+
31+
#ifndef GPUCA_GPUCODE_DEVICE
32+
/// Majority-vote MC label of a vertex: the most frequent (source, event) among its
33+
/// contributors, flagged fake when no label reaches more than half of them. Templated
34+
/// on the container so both bounded_vector and std::vector callers share one copy
35+
template <typename Container>
36+
VertexLabel computeMainVertexLabel(const Container& elements)
37+
{
38+
// we only care about the source&event of the tracks, not the trackId
39+
auto composeVtxLabel = [](const o2::MCCompLabel& lbl) -> o2::MCCompLabel {
40+
return {o2::MCCompLabel::maxTrackID(), lbl.getEventID(), lbl.getSourceID(), lbl.isFake()};
41+
};
42+
std::unordered_map<o2::MCCompLabel, size_t> frequency;
43+
for (const auto& element : elements) {
44+
++frequency[composeVtxLabel(element)];
45+
}
46+
o2::MCCompLabel elem{};
47+
size_t maxCount = 0;
48+
for (const auto& [key, count] : frequency) {
49+
if (count > maxCount) {
50+
maxCount = count;
51+
elem = key;
52+
}
53+
}
54+
if (maxCount <= 1) { // need >50%
55+
elem.setFakeFlag();
56+
}
57+
return std::make_pair(elem, static_cast<float>(maxCount) / static_cast<float>(elements.size()));
58+
}
59+
#endif
2860
} // namespace o2::its
2961

3062
#ifndef GPUCA_GPUCODE_DEVICE

‎Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h‎

Lines changed: 108 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
#include "ITStracking/Configuration.h"
2222
#include "ITStracking/TrackExtensionHypothesis.h"
2323
#include "ITStrackingGPU/Utils.h"
24+
#include "ITStracking/ClusterLines.h"
25+
#include "ITStracking/LineProjection.h"
2426

2527
namespace o2::its::gpu
2628
{
@@ -54,10 +56,14 @@ class TimeFrameGPU : public TimeFrame<NLayers>
5456
void createTrackingFrameInfoDeviceArray(const int = NLayers);
5557
void loadUnsortedClustersDevice(const int);
5658
void createUnsortedClustersDeviceArray(const int = NLayers);
57-
void loadClustersDevice(const int);
5859
void createClustersDeviceArray(const int = NLayers);
5960
void loadClustersIndexTables(const int);
6061
void createClustersIndexTablesArray(const int = NLayers);
62+
void createClustersDevice(const int);
63+
void createClustersIndexTables(const int);
64+
void createClusterRadiiDevice();
65+
void uploadClusterRadii();
66+
void sortClustersDevice(const int layer, const TrackingParameters& trkParam);
6167
void createUsedClustersDevice(const int);
6268
void createUsedClustersDeviceArray(const int = NLayers);
6369
void loadUsedClustersDevice();
@@ -87,6 +93,20 @@ class TimeFrameGPU : public TimeFrame<NLayers>
8793
void createTrackExtensionScratchDevice(const int nThreads, const int maxHypotheses);
8894
void downloadTrackITSExtDevice();
8995

96+
// Seeding-vertexer
97+
void createClusterOwnersDeviceArray();
98+
void createClusterOwnersDevice();
99+
void resetClusterOwnersDevice();
100+
void createClusterSortScratchDevice(const int layer);
101+
102+
void createLinesDevice(const int nCells);
103+
void createDiamondDevice(const Vertex& diamond);
104+
unsigned int downloadLinesDevice();
105+
unsigned int getNLines();
106+
const auto& getHostLines() const { return mLinesHost; }
107+
const auto& getHostLineRof() const { return mLineRofHost; }
108+
const auto& getHostLineClusters() const { return mLineClustersHost; }
109+
90110
/// synchronization
91111
auto& getStream(const size_t stream) { return mGpuStreams[stream]; }
92112
auto& getStreams() { return mGpuStreams; }
@@ -111,6 +131,17 @@ class TimeFrameGPU : public TimeFrame<NLayers>
111131
auto& getTrackITSExt() { return mTrackITSExt; }
112132
auto& getTrackIndices() { return mTrackIndices; }
113133
Vertex* getDeviceVertices() { return mPrimaryVerticesDevice; }
134+
int* getDeviceROFramesClusters(const int layer) { return mROFramesClustersDevice[layer]; }
135+
int* getDeviceClusterSortKeys(const int layer) { return mClusterSortKeysDevice[layer]; }
136+
int* getDeviceClusterSortPerm(const int layer) { return mClusterSortPermDevice[layer]; }
137+
Cluster* getDeviceUnsortedClusters(const int layer) { return mUnsortedClustersDevice[layer]; }
138+
Cluster* getDeviceClusters(const int layer) { return mClustersDevice[layer]; }
139+
int* getDeviceClustersIndexTable(const int layer) { return mClustersIndexTablesDevice[layer]; }
140+
const float* getDeviceMinRs() const { return mClusterMinRDevice; }
141+
const float* getDeviceMaxRs() const { return mClusterMaxRDevice; }
142+
int* getDeviceROFramesPV() { return mROFramesPVDevice; }
143+
unsigned char* getDeviceUsedClusters(const int);
144+
const o2::base::Propagator* getChainPropagator();
114145

115146
// Hybrid
116147
TrackITSExt* getDeviceTrackITSExt() { return mTrackITSExtDevice; }
@@ -119,6 +150,39 @@ class TimeFrameGPU : public TimeFrame<NLayers>
119150
TrackExtensionHypothesis<NLayers>* getDeviceNextTrackExtensionHypotheses() { return mNextTrackExtensionHypothesesDevice; }
120151
int* getDeviceNeighboursLUT(const int layer) { return mNeighboursLUTDevice[layer]; }
121152
CellNeighbour** getDeviceArrayNeighbours() { return mNeighboursDeviceArray; }
153+
unsigned long long** getDeviceArrayClusterOwners() { return mClusterOwnersDeviceArray; }
154+
o2::its::Line* getDeviceLines() { return mLinesDevice; }
155+
int* getDeviceLineSlots() { return mLineSlotsDevice; }
156+
int* getDeviceLineRof() { return mLineRofDevice; }
157+
int* getDeviceLineClusters() { return mLineClustersDevice; }
158+
float* getDeviceLineChi2() { return mLineChi2Device; }
159+
float* getDeviceLinePt() { return mLinePtDevice; }
160+
float* getDeviceLineZs() { return mLineZsDevice; }
161+
o2::its::TimeEstBC* getDeviceLineTimes() { return mLineTimesDevice; }
162+
int* getDeviceLineSortedIdx() { return mLinesSortedIdx; }
163+
LineProjSoA getLineProjSoA() { return {mLineZsDevice, mLineTimesDevice, mLinesSortedIdx, mLineRofDevice}; }
164+
LineProjSoA getLineProjSortedSoA() { return {mLineZsSortedDevice, mLineTimesSortedDevice, mLinesSortedIdx, mLineRofSortedDevice}; }
165+
int* getDeviceRofLineOffsets() { return mRofLineOffsetsDevice; }
166+
int* getDeviceLineDensity() { return mLineDensityDevice; }
167+
gpu::LineWindow* getDeviceLineWin() { return mLineWinDevice; }
168+
uint8_t* getDeviceLineIsPeak() { return mLineIsPeakDevice; }
169+
int* getDeviceLineDensityFine() { return mLineDensityFineDevice; }
170+
gpu::LineWindow* getDeviceLineWinFine() { return mLineWinFineDevice; }
171+
uint8_t* getDeviceLineIsPeakFine() { return mLineIsPeakFineDevice; }
172+
int* getDevicePeakScan() { return mPeakScanDevice; }
173+
int* getDevicePeakLineIdx() { return mPeakLineIdxDevice; }
174+
int* getDevicePeakOffsets() { return mPeakOffsetsDevice; }
175+
const int* getDeviceNPeaks() { return mPeakOffsetsDevice + this->getNrof(1); }
176+
VertexCand* getDeviceVertexCands() { return mVertexCandsDevice; }
177+
int downloadVertexCandsDevice();
178+
void downloadPeakMembershipInputs(); // MC-only: peak indices, z-windows and the sorted time/idx columns
179+
const auto& getHostVertexCands() const { return mVertexCandsHost; }
180+
const auto& getHostPeakOffsets() const { return mPeakOffsetsHost; }
181+
const auto& getHostPeakMembership() const { return mPeakMembershipHost; }
182+
std::vector<o2::MCCompLabel>& getLineLabelFlat() { return mLineLabelFlatHost; }
183+
const std::vector<o2::MCCompLabel>& getLineLabelFlat() const { return mLineLabelFlatHost; }
184+
Vertex* getDeviceDiamond() { return mDiamondDevice; }
185+
std::array<CellNeighbour*, MaxCells>& getDeviceNeighboursAll() { return mNeighboursDevice; }
122186
CellNeighbour* getDeviceNeighbours(const int layer) { return mNeighboursDevice[layer]; }
123187
const TrackingFrameInfo** getDeviceArrayTrackingFrameInfo() const { return mTrackingFrameInfoDeviceArray; }
124188
const Cluster** getDeviceArrayClusters() const { return mClustersDeviceArray; }
@@ -156,6 +220,10 @@ class TimeFrameGPU : public TimeFrame<NLayers>
156220
size_t getNumberOfCells() const final;
157221
size_t getNumberOfNeighbours() const final;
158222

223+
protected:
224+
void prepareClusters(const TrackingParameters& trkParam, const int maxLayers) override;
225+
void allocateClusterSortStorage(const TrackingParameters& trkParam, const int maxLayers) override;
226+
159227
private:
160228
enum class SlotInit {
161229
Raw, ///< whatever the allocator handed back
@@ -215,6 +283,11 @@ class TimeFrameGPU : public TimeFrame<NLayers>
215283
const int** mClustersIndexTablesDeviceArray{nullptr};
216284
uint8_t** mUsedClustersDeviceArray{nullptr};
217285
const int** mROFramesClustersDeviceArray{nullptr};
286+
int* mROFramesPVDevice;
287+
std::array<int*, NLayers> mClusterSortKeysDevice{};
288+
std::array<int*, NLayers> mClusterSortPermDevice{};
289+
float* mClusterMinRDevice{nullptr};
290+
float* mClusterMaxRDevice{nullptr};
218291
std::array<Tracklet*, MaxLinks> mTrackletsDevice{};
219292
std::array<int*, MaxLinks> mTrackletsLUTDevice{};
220293
std::array<int*, MaxCells> mCellsLUTDevice{};
@@ -239,6 +312,40 @@ class TimeFrameGPU : public TimeFrame<NLayers>
239312
CellNeighbour** mNeighboursDeviceArray{nullptr};
240313
std::array<TrackingFrameInfo*, NLayers> mTrackingFrameInfoDevice{};
241314
const TrackingFrameInfo** mTrackingFrameInfoDeviceArray{nullptr};
315+
std::array<unsigned long long*, 3> mClusterOwnersDevice{};
316+
unsigned long long** mClusterOwnersDeviceArray{nullptr};
317+
int* mLineSlotsDevice{nullptr};
318+
o2::its::Line* mLinesDevice{nullptr};
319+
int* mLineRofDevice{nullptr};
320+
int* mLineClustersDevice{nullptr};
321+
float* mLineChi2Device{nullptr};
322+
float* mLinePtDevice{nullptr};
323+
float* mLineZsDevice{nullptr};
324+
o2::its::TimeEstBC* mLineTimesDevice{nullptr};
325+
float* mLineZsSortedDevice{nullptr};
326+
o2::its::TimeEstBC* mLineTimesSortedDevice{nullptr};
327+
int* mLinesSortedIdx{nullptr};
328+
int* mLineRofSortedDevice{nullptr}; // per (sorted) line's ROF
329+
int* mRofLineOffsetsDevice{nullptr}; // CSR offsets into the (rof,z)-sorted lines, size nRofs+1
330+
int* mLineDensityDevice{nullptr}; // per (sorted) line: count of time-compatible neighbours in its z-window
331+
gpu::LineWindow* mLineWinDevice{nullptr}; // per (sorted) line: [lo,hi) bounds of its z-window (sorted coords)
332+
uint8_t* mLineIsPeakDevice{nullptr}; // per (sorted) line: 1 if it is a local density peak (vertex candidate)
333+
int* mLineDensityFineDevice{nullptr};
334+
gpu::LineWindow* mLineWinFineDevice{nullptr};
335+
uint8_t* mLineIsPeakFineDevice{nullptr};
336+
int* mPeakScanDevice{nullptr}; // per (sorted) line: number of peaks strictly before it
337+
int* mPeakLineIdxDevice{nullptr}; // per peak slot: the sorted line index it came from
338+
int* mPeakOffsetsDevice{nullptr}; // CSR offsets into the compacted peaks
339+
VertexCand* mVertexCandsDevice{nullptr};
340+
int mNLinesCapacity{0}; // = nCells the line buffers were sized for
341+
std::vector<o2::its::Line> mLinesHost;
342+
std::vector<int> mLineRofHost;
343+
std::vector<int> mLineClustersHost;
344+
std::vector<VertexCand> mVertexCandsHost;
345+
std::vector<int> mPeakOffsetsHost;
346+
PeakMembershipHost mPeakMembershipHost;
347+
std::vector<o2::MCCompLabel> mLineLabelFlatHost;
348+
Vertex* mDiamondDevice{nullptr};
242349

243350
// State
244351
Streams mGpuStreams;

‎Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackerTraitsGPU.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@ class TrackerTraitsGPU final : public TrackerTraits<NLayers>
2929
void adoptTimeFrame(TimeFrame<NLayers>* tf) final;
3030
void initialiseTimeFrame(const int iteration) final;
3131

32+
void computeVertexCandidates(const int iteration) final;
33+
void computeVertices(const int iteration) final;
34+
3235
void computeLayerTracklets(const int iteration, int) final;
3336
void computeLayerCells(const int iteration) final;
3437
void findCellsNeighbours(const int iteration) final;

‎Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h‎

Lines changed: 108 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
#include "ITStracking/TrackingTopology.h"
2323
#include "ITStracking/TrackExtensionHypothesis.h"
2424
#include "ITStrackingGPU/Utils.h"
25+
#include "ITStracking/ClusterLines.h"
26+
#include "ITStracking/LineProjection.h"
2527
#include "DetectorsBase/Propagator.h"
2628

2729
namespace o2::its
@@ -52,6 +54,7 @@ struct TrackingKernels {
5254
const typename ROFVertexLookupTable<NLayers>::View& vertexLUT,
5355
const int vertexId,
5456
const Vertex* vertices,
57+
const bool vtxMode,
5558
const Cluster** clusters,
5659
const std::vector<unsigned int>& nClusters,
5760
const int** ROFClusters,
@@ -67,8 +70,8 @@ struct TrackingKernels {
6770
const typename TrackingTopology<NLayers>::View topology,
6871
bounded_vector<float>& linkPhiCuts,
6972
const float resolutionPV,
70-
std::array<float, NLayers>& minR,
71-
std::array<float, NLayers>& maxR,
73+
const float* minRs,
74+
const float* maxRs,
7275
bounded_vector<float>& resolutions,
7376
std::vector<float>& radii,
7477
bounded_vector<float>& linkMSAngles,
@@ -89,6 +92,7 @@ struct TrackingKernels {
8992
const float bz,
9093
const float maxChi2ClusterAttachment,
9194
const float cellDeltaTanLambdaSigma,
95+
const float cellDeltaPhiCut,
9296
const float nSigmaCut,
9397
const float* layerxX0,
9498
o2::its::ExternalAllocator* alloc,
@@ -172,6 +176,108 @@ struct TrackingKernels {
172176
const o2::base::Propagator* propagator,
173177
const o2::base::PropagatorF::MatCorrType matCorrType,
174178
o2::its::ExternalAllocator* alloc);
179+
180+
static void sortClustersHandler(const Cluster* unsorted,
181+
Cluster* sorted,
182+
const int* clusterOffsets,
183+
int* indexTable,
184+
const IndexTableUtils<NLayers>* utils,
185+
const typename ROFMaskTable<NLayers>::View& rofMask,
186+
float beamX, float beamY,
187+
int zBins, int phiBins, int nRofs, int nClustersLayer, int iLayer,
188+
float* minRadiusLayer, float* maxRadiusLayer,
189+
int* keys,
190+
int* perm,
191+
o2::its::ExternalAllocator* alloc,
192+
gpu::Stream& stream);
193+
194+
static void registerClusterOwnershipHandler(const CellSeed* cellsLayersDevice,
195+
const int nCells,
196+
unsigned long long** clusterOwnersDeviceArray,
197+
gpu::Stream& stream);
198+
199+
static void linearizeCellsToLinesHandler(const int nCells,
200+
const CellSeed* cells,
201+
const unsigned long long* const* clusterOwners,
202+
const int* rofFramesClustersL1,
203+
const int nRofsL1,
204+
const int ownedClustersCut,
205+
o2::its::Line* lines,
206+
int* lineRof,
207+
int* lineClusters,
208+
int* lineSlots,
209+
const float beamX,
210+
const float beamY,
211+
const float maxZ,
212+
const float minPt,
213+
float* linesZs,
214+
o2::its::TimeEstBC* lineTimes,
215+
float* lineChi2,
216+
float* linePt,
217+
o2::its::ExternalAllocator* alloc,
218+
gpu::Stream& stream);
219+
220+
static void sortLinesHandler(const int nLines,
221+
const int nRofs,
222+
const gpu::LineProjSoA soa,
223+
const gpu::LineProjSoA sortedSoa,
224+
const int* lineRof,
225+
int* rofOffsets,
226+
o2::its::ExternalAllocator* alloc,
227+
gpu::Stream& stream);
228+
229+
static void scanDensityHandler(const int nLines,
230+
const gpu::LineProjSoA sortedSoa,
231+
const int* rofOffsets,
232+
int* density,
233+
gpu::LineWindow* win,
234+
const float zWindow,
235+
gpu::Stream& stream);
236+
237+
static void findPeaksHandler(const int nLines,
238+
const int nRofs,
239+
const gpu::LineProjSoA sortedSoa,
240+
const int* rofOffsets,
241+
const int* density,
242+
const gpu::LineWindow* win,
243+
uint8_t* isPeak,
244+
const int* densityFine,
245+
const gpu::LineWindow* winFine,
246+
const int fineMinDensity,
247+
uint8_t* isPeakFine,
248+
int* peakScan,
249+
int* peakLineIdx,
250+
int* peakOffsets,
251+
o2::its::ExternalAllocator* alloc,
252+
gpu::Stream& stream);
253+
254+
static void fitPeaksHandler(const int* nPeaksDevice,
255+
const int* peakLineIdx,
256+
const gpu::LineWindow* win,
257+
const gpu::LineProjSoA sortedSoa,
258+
const o2::its::Line* lines,
259+
const float* lineChi2,
260+
const float* linePt,
261+
const float goodLineChi2Cut,
262+
const float goodLinePtCut,
263+
const float pairCut2,
264+
const float nSigmaCut,
265+
const int minContributors,
266+
const float beamX,
267+
const float beamY,
268+
const uint8_t* isPeakFine,
269+
const float fineMaxDrift,
270+
gpu::VertexCand* cands,
271+
gpu::Stream& stream);
272+
273+
static void dedupVertexCandidatesHandler(const int* nPeaksDevice,
274+
const int* peakLineIdx,
275+
const int* peakOffsets,
276+
const gpu::LineProjSoA sortedSoa,
277+
const float duplicateZCut,
278+
const float duplicateZScale,
279+
gpu::VertexCand* cands,
280+
gpu::Stream& stream);
175281
};
176282

177283
void resetOutputCounterHandler(int* outputCounter, gpu::Stream& stream);

0 commit comments

Comments
 (0)