32 #include "ReductionMgr.decl.h"
36 #define MIN_DEBUG_LEVEL 4
60 NAMD_bug(
"ReductionSet size specified for REDUCTIONS_BASIC or REDUCTIONS_AMD.");
64 if ( size == -1 )
NAMD_bug(
"ReductionSet size not specified.");
93 if ( (*current)->sequenceNumber == seqNum )
return *current;
94 current = &((*current)->next);
108 if ( (*current)->sequenceNumber == seqNum )
break;
109 current = &((*current)->next);
112 if ( ! *current ) {
NAMD_die(
"ReductionSet::removeData on missing seqNum"); }
115 *current = (*current)->
next;
120 const int max_intranode_children,
121 const int max_internode_children,
132 const int num_pes = CkNumPes();
133 const int num_node_pes = CmiNumPesOnPhysicalNode(CmiPhysicalNodeID(pe));
134 int *node_pes =
new int[num_node_pes];
136 const int first_pe = CmiGetFirstPeOnPhysicalNode(CmiPhysicalNodeID(pe));
138 int *node_ids =
new int[num_pes];
139 int first_pe_index = -1;
143 if (pe == 0 && first_pe != pe) {
144 NAMD_die(
"PE 0 is not the first physical node. This shouldn't happen");
149 for (i = 0; i < num_pes; i++) {
151 if (CmiGetFirstPeOnPhysicalNode(CmiPhysicalNodeID(i)) == i) {
152 node_ids[num_nodes] = i;
154 first_pe_index = num_nodes;
159 const int i1 = (i + first_pe) % num_pes;
160 if (CmiPeOnSamePhysicalNode(first_pe,i1)) {
161 if ( node_pe_count == num_node_pes )
162 NAMD_bug(
"ReductionMgr::buildSpanTree found inconsistent physical node data from Charm++ runtime");
163 node_pes[node_pe_count] = i1;
165 pe_index = node_pe_count;
169 if ( pe_index < 0 || first_pe_index < 0 )
170 NAMD_bug(
"ReductionMgr::buildSpanTree found inconsistent physical node data from Charm++ runtime");
175 int first_loc_child_index = pe_index * max_intranode_children + 1;
176 int last_loc_child_index
177 = first_loc_child_index + max_intranode_children - 1;
178 if (first_loc_child_index > num_node_pes) {
179 first_loc_child_index = num_node_pes;
180 last_loc_child_index = num_node_pes;
182 if (last_loc_child_index >= num_node_pes)
183 last_loc_child_index = num_node_pes-1;
189 int first_rem_child_index = num_nodes;
190 int last_rem_child_index = num_nodes;
192 int *rem_child_index =
new int[max_internode_children];
194 if (first_pe != pe) {
197 my_parent_index = (pe_index-1)/max_intranode_children;
198 *parent = node_pes[my_parent_index];
204 int range_end = num_nodes;
207 my_parent_index = -1;
211 while ( first_pe_index != range_begin ) {
212 my_parent_index = range_begin;
214 for (
int i = 0; i < max_internode_children; ++i ) {
215 int split = range_begin + ( range_end - range_begin ) / ( max_internode_children - i );
216 if ( first_pe_index < split ) { range_end =
split;
break; }
217 else { range_begin =
split; }
220 *parent = node_ids[my_parent_index];
224 int prev_child_index = range_begin;
226 for (
int i = 0; i < max_internode_children; ++i ) {
227 if ( range_begin >= range_end )
break;
228 if ( range_begin > prev_child_index ) {
229 rem_child_index[rem_children++] = prev_child_index = range_begin;
231 range_begin += ( range_end - range_begin ) / ( max_internode_children - i );
239 if (first_loc_child_index != num_node_pes) {
240 loc_children = last_loc_child_index - first_loc_child_index + 1;
241 *num_children += loc_children;
248 *num_children += rem_children;
253 if (*num_children == 0)
256 *children =
new int[*num_children];
260 if (loc_children > 0) {
261 for(k=first_loc_child_index; k <= last_loc_child_index; k++) {
263 (*children)[child++]=node_pes[k];
266 if (rem_children > 0) {
267 for(k=0; k < rem_children; k++) {
269 (*children)[child++]=node_ids[rem_child_index[k]];
273 delete [] rem_child_index;
280 if (CkpvAccess(ReductionMgr_instance) == 0) {
281 CkpvAccess(ReductionMgr_instance) =
this;
283 DebugM(1,
"ReductionMgr::ReductionMgr() - another instance exists!\n");
287 &myParent,&numChildren,&children);
298 #if 0 // Old spanning tree
305 if (firstChild > CkNumPes()) firstChild = CkNumPes();
307 if (lastChild > CkNumPes()) lastChild = CkNumPes();
312 reductionSets[i] = 0;
315 DebugM(1,
"ReductionMgr() instantiated.\n");
323 delete reductionSets[i];
329 ReductionSet* ReductionMgr::getSet(
int setID,
int size) {
330 if ( reductionSets[setID] == 0 ) {
331 reductionSets[setID] =
new ReductionSet(setID,size,numChildren);
337 CProxy_ReductionMgr reductionProxy(thisgroup);
338 reductionProxy[myParent].remoteRegister(msg);
341 if ( size != -1 )
NAMD_bug(
"ReductionMgr::getSet size set");
342 }
else if ( size < 0 || reductionSets[setID]->dataSize != size ) {
343 NAMD_bug(
"ReductionMgr::getSet size mismatch");
345 return reductionSets[setID];
349 void ReductionMgr::delSet(
int setID) {
356 CProxy_ReductionMgr reductionProxy(thisgroup);
357 reductionProxy[myParent].remoteUnregister(msg);
360 reductionSets[setID] = 0;
369 NAMD_die(
"ReductionMgr::willSubmit called while reductions outstanding!");
375 handle->reductionSetID = setID;
377 handle->master =
this;
378 handle->data = data->
data;
385 int setID = handle->reductionSetID;
388 NAMD_die(
"SubmitReduction deleted while reductions outstanding!");
398 int setID = handle->reductionSetID;
399 int seqNum = handle->sequenceNumber;
405 mergeAndDeliver(set,seqNum);
408 handle->sequenceNumber = ++seqNum;
419 NAMD_die(
"ReductionMgr::remoteRegister called while reductions outstanding on parent!");
437 NAMD_die(
"SubmitReduction deleted while reductions outstanding on parent!");
456 NAMD_bug(
"ReductionMgr::remoteSubmit data sizes do not match.");
463 #pragma disjoint (*curData, *newData)
467 for (
int i = 0; i < size; ++i ) {
468 if ( newData[i] > curData[i] ) {
469 curData[i] = newData[i];
473 for (
int i = 0; i < size; ++i ) {
474 curData[i] += newData[i];
483 mergeAndDeliver(set,seqNum);
488 void ReductionMgr::mergeAndDeliver(
ReductionSet *set,
int seqNum) {
496 NAMD_bug(
"ReductionMgr::mergeAndDeliver not ready to deliver.");
506 NAMD_die(
"ReductionSet::deliver will never deliver data");
515 for (
int i = 0; i < msg->
dataSize; ++i ) {
518 CProxy_ReductionMgr reductionProxy(thisgroup);
519 reductionProxy[myParent].remoteSubmit(msg);
530 NAMD_die(
"ReductionMgr::willRequire called while reductions outstanding!");
534 handle->reductionSetID = setID;
536 handle->master =
this;
543 int setID = handle->reductionSetID;
546 NAMD_die(
"RequireReduction deleted while reductions outstanding!");
556 int setID = handle->reductionSetID;
558 int seqNum = handle->sequenceNumber;
570 delete handle->currentData;
571 handle->currentData = set->
removeData(seqNum);
572 handle->data = handle->currentData->
data;
573 handle->sequenceNumber = ++seqNum;
577 #include "ReductionMgr.def.h"
void buildSpanTree(const int pe, const int max_intranode_children, const int max_internode_children, int *parent, int *num_children, int **children)
int * addToRemoteSequenceNumber
ReductionSetData * getData(int seqNum)
ReductionSet(int setID, int size, int numChildren)
void remoteRegister(ReductionRegisterMsg *msg)
SubmitReduction * willSubmit(int setID, int size=-1)
friend class SubmitReduction
#define REDUCTION_MAX_CHILDREN
static Units next(Units u)
void NAMD_bug(const char *err_msg)
int waitingForSequenceNumber
ReductionSetData * removeData(int seqNum)
void NAMD_die(const char *err_msg)
std::vector< std::string > split(const std::string &text, std::string delimiter)
void remoteSubmit(ReductionSubmitMsg *msg)
RequireReduction * willRequire(int setID, int size=-1)
friend class RequireReduction
void remoteUnregister(ReductionRegisterMsg *msg)
ReductionSetData * dataQueue