Modularize the Arkouda build process

.github/workflows/CI.yml

@@ -132,4 +132,5 @@ jobs:
         echo "\$(eval \$(call add-path,/usr/lib/x86_64-linux-gnu/hdf5/serial/))" > Makefile.paths
     - name: Run unit tests
       run: |
+        python3 src/serverModuleGen.py ServerModules.cfg
         start_test test

Makefile

@@ -62,6 +62,10 @@ ifdef ARKOUDA_ARROW_PATH
 $(eval $(call add-path,$(ARKOUDA_ARROW_PATH)))
 endif
 
+ifndef ARKOUDA_CONFIG_FILE
+ARKOUDA_CONFIG_FILE := $(ARKOUDA_PROJECT_DIR)/ServerModules.cfg
+endif
+
 CHPL_FLAGS += -lhdf5 -lhdf5_hl -lzmq
 
 ifdef ARKOUDA_SERVER_PARQUET_SUPPORT
@@ -252,8 +256,10 @@ else
 	ARKOUDA_COMPAT_MODULES += -M $(ARKOUDA_SOURCE_DIR)/compat/lt-125
 endif
 
+MODULE_GENERATION_SCRIPT=$(ARKOUDA_SOURCE_DIR)/serverModuleGen.py
 # This is the main compilation statement section
 $(ARKOUDA_MAIN_MODULE): check-deps $(ARROW_O) $(ARKOUDA_SOURCES) $(ARKOUDA_MAKEFILES)
+	python3 $(MODULE_GENERATION_SCRIPT) $(ARKOUDA_CONFIG_FILE)
 	$(CHPL) $(CHPL_DEBUG_FLAGS) $(PRINT_PASSES_FLAGS) $(REGEX_MAX_CAPTURES_FLAG) $(OPTIONAL_SERVER_FLAGS) $(CHPL_FLAGS_WITH_VERSION) $(ARKOUDA_MAIN_SOURCE) $(ARKOUDA_COMPAT_MODULES) -o $@
 
 CLEAN_TARGETS += arkouda-clean

ServerModules.cfg

@@ -0,0 +1,28 @@
+# Comment out modules using a `#` to exclude
+# a module from a build
+
+ArraySetopsMsg
+KExtremeMsg
+ArgSortMsg
+SegmentedMsg
+OperatorMsg
+RandMsg
+IndexingMsg
+UniqueMsg
+In1dMsg
+HistogramMsg
+SortMsg
+ReductionMsg
+FindSegmentsMsg
+EfuncMsg
+ConcatenateMsg
+JoinEqWithDTMsg
+RegistrationMsg
+CastMsg
+BroadcastMsg
+FlattenMsg
+
+# Add additional modules below
+# For modules not in the Arkouda `src/` directory,
+# make sure to include the full path to the module.
+# Example: `/Users/my-username/MyModule.chpl`

src/.gitignore

@@ -1,2 +1,3 @@
 TAGS
 ArrowFunctions.o
+ServerRegistration.chpl

src/ArgSortMsg.chpl

@@ -485,4 +485,10 @@ module ArgSortMsg
         asLogger.debug(getModuleName(),getRoutineName(),getLineNumber(),repMsg);
         return new MsgTuple(repMsg, MsgType.NORMAL);
     }
+
+    proc registerMe() {
+      use CommandMap;
+      registerFunction("argsort", argsortMsg);
+      registerFunction("coargsort", coargsortMsg);
+    }
 }

src/ArraySetopsMsg.chpl

@@ -237,4 +237,12 @@ module ArraySetopsMsg
             moduleName=getModuleName(),
             errorClass="ErrorWithContext");
     }
+
+    proc registerMe() {
+      use CommandMap;
+      registerFunction("intersect1d", intersect1dMsg);
+      registerFunction("setdiff1d", setdiff1dMsg);
+      registerFunction("setxor1d", setxor1dMsg);
+      registerFunction("union1d", union1dMsg);
+    }
 }

src/BroadcastMsg.chpl

@@ -110,4 +110,9 @@ module BroadcastMsg {
     bmLogger.debug(getModuleName(),getRoutineName(),getLineNumber(),repMsg);
     return new MsgTuple(repMsg, MsgType.NORMAL);    
   }
+
+  proc registerMe() {
+    use CommandMap;
+    registerFunction("broadcast", broadcastMsg);
+  }
 }

src/CastMsg.chpl

@@ -214,5 +214,9 @@ module CastMsg {
       castLogger.debug(getModuleName(),getRoutineName(),getLineNumber(),returnMsg);
       return returnMsg;
   }
-
+
+  proc registerMe() {
+    use CommandMap;
+    registerFunction("cast", castMsg);
+  }
 }

src/CommandMap.chpl

@@ -0,0 +1,22 @@
+module CommandMap {
+  use ServerRegistration;
+  use Map;
+  use Message;
+  use MultiTypeSymbolTable;
+
+  // This is a dummy function to get the signature of the Arkouda
+  // server FCF. Ideally, the `func()` function would be able to
+  // construct the FCF type, but there is no way to generate a
+  // FCF that throws using `func()` today.
+  proc akMsgSign(a: string, b: string, c: borrowed SymTab): MsgTuple throws {
+    var rep = new MsgTuple("dummy-msg", MsgType.NORMAL);
+    return rep;
+  }
+
+  var f = akMsgSign;
+  var commandMap: map(string, f.type);
+
+  proc registerFunction(cmd: string, fcf: f.type) {
+    commandMap.add(cmd, fcf);
+  }
+}

src/ConcatenateMsg.chpl

@@ -326,4 +326,9 @@ module ConcatenateMsg
             }
         }
     }
+
+    proc registerMe() {
+      use CommandMap;
+      registerFunction("concatenate", concatenateMsg);
+    }
 }

src/EfuncMsg.chpl

@@ -692,4 +692,12 @@ module EfuncMsg
       return C;
     }    
 
+    proc registerMe() {
+      use CommandMap;
+      registerFunction("efunc", efuncMsg);
+      registerFunction("efunc3vv", efunc3vvMsg);
+      registerFunction("efunc3vs", efunc3vsMsg);
+      registerFunction("efunc3sv", efunc3svMsg);
+      registerFunction("efunc3ss", efunc3ssMsg);
+    }
 }

src/FindSegmentsMsg.chpl

@@ -204,4 +204,9 @@ module FindSegmentsMsg
         fsLogger.debug(getModuleName(),getRoutineName(),getLineNumber(),repMsg);
         return new MsgTuple(repMsg, MsgType.NORMAL);
     }
+
+    proc registerMe() {
+      use CommandMap;
+      registerFunction("findSegments", findSegmentsMsg);
+    }
 }

src/FlattenMsg.chpl

@@ -94,4 +94,10 @@ module FlattenMsg {
     fmLogger.debug(getModuleName(),getRoutineName(),getLineNumber(),repMsg);
     return new MsgTuple(repMsg, MsgType.NORMAL);
   }
+
+  proc registerMe() {
+    use CommandMap;
+    registerFunction("segmentedFlatten", segFlattenMsg);
+    registerFunction("segmentedSplit", segmentedSplitMsg);
+  }
 }

src/HistogramMsg.chpl

@@ -77,4 +77,9 @@ module HistogramMsg
         hgmLogger.debug(getModuleName(),getRoutineName(),getLineNumber(),repMsg);
         return new MsgTuple(repMsg, MsgType.NORMAL);
     }
+
+    proc registerMe() {
+      use CommandMap;
+      registerFunction("histogram", histogramMsg);
+    }
 }

src/In1dMsg.chpl

@@ -104,5 +104,10 @@ module In1dMsg
         repMsg = "created " + st.attrib(rname);
         iLogger.debug(getModuleName(),getRoutineName(),getLineNumber(),repMsg);
         return new MsgTuple(repMsg, MsgType.NORMAL);
-    }   
+    }
+
+    proc registerMe() {
+      use CommandMap;
+      registerFunction("in1d", in1dMsg);
+    }
 }

src/IndexingMsg.chpl

@@ -719,5 +719,17 @@ module IndexingMsg
         repMsg = "%s success".format(pn);
         imLogger.debug(getModuleName(),getRoutineName(),getLineNumber(),repMsg);
         return new MsgTuple(repMsg, MsgType.NORMAL); 
-    } 
+    }
+
+    proc registerMe() {
+      use CommandMap;
+      registerFunction("[int]", intIndexMsg);
+      registerFunction("[slice]", sliceIndexMsg);
+      registerFunction("[pdarray]", pdarrayIndexMsg);
+      registerFunction("[int]=val", setIntIndexToValueMsg);
+      registerFunction("[pdarray]=val", setPdarrayIndexToValueMsg);
+      registerFunction("[pdarray]=pdarray", setPdarrayIndexToPdarrayMsg);
+      registerFunction("[slice]=val", setSliceIndexToValueMsg);
+      registerFunction("[slice]=pdarray", setSliceIndexToPdarrayMsg);
+    }
 }

src/JoinEqWithDTMsg.chpl

@@ -369,5 +369,9 @@ module JoinEqWithDTMsg
         jeLogger.debug(getModuleName(),getRoutineName(),getLineNumber(),repMsg);
         return new MsgTuple(repMsg, MsgType.NORMAL);
     }// end joinEqWithDTMsg()
-
+
+    proc registerMe() {
+      use CommandMap;
+      registerFunction("joinEqWithDT", joinEqWithDTMsg);
+    }
 }// end module JoinEqWithDTMsg

src/KExtremeMsg.chpl

@@ -151,4 +151,10 @@ module KExtremeMsg
            }
         }
     }
+
+    proc registerMe() {
+      use CommandMap;
+      registerFunction("mink", minkMsg);
+      registerFunction("maxk", maxkMsg);
+    }
 }

src/MsgProcessing.chpl

@@ -16,27 +16,6 @@ module MsgProcessing
 
     use AryUtil;
 
-    public use OperatorMsg;
-    public use RandMsg;
-    public use IndexingMsg;
-    public use UniqueMsg;
-    public use In1dMsg;
-    public use HistogramMsg;
-    public use ArgSortMsg;
-    public use SortMsg;
-    public use ReductionMsg;
-    public use FindSegmentsMsg;
-    public use EfuncMsg;
-    public use ConcatenateMsg;
-    public use SegmentedMsg;
-    public use JoinEqWithDTMsg;
-    public use RegistrationMsg;
-    public use ArraySetopsMsg;
-    public use KExtremeMsg;
-    public use CastMsg;
-    public use BroadcastMsg;
-    public use FlattenMsg;
-
     private config const logLevel = ServerConfig.logLevel;
     const mpLogger = new Logger(logLevel);
 

src/OperatorMsg.chpl

@@ -1696,4 +1696,13 @@ module OperatorMsg
         omLogger.debug(getModuleName(),getRoutineName(),getLineNumber(),repMsg);
         return new MsgTuple(repMsg, MsgType.NORMAL);
     }
+
+    proc registerMe() {
+      use CommandMap;
+      registerFunction("binopvv", binopvvMsg);
+      registerFunction("binopvs", binopvsMsg);
+      registerFunction("binopsv", binopsvMsg);
+      registerFunction("opeqvv", opeqvvMsg);
+      registerFunction("opeqvs", opeqvsMsg);
+    }
 }

src/RandMsg.chpl

@@ -120,4 +120,10 @@ module RandMsg
         randLogger.debug(getModuleName(),getRoutineName(),getLineNumber(),repMsg);
         return new MsgTuple(repMsg, MsgType.NORMAL);
     }
+
+    proc registerMe() {
+      use CommandMap;
+      registerFunction("randint", randintMsg);
+      registerFunction("randomNormal", randomNormalMsg);
+    }
 }

src/ReductionMsg.chpl

@@ -961,5 +961,12 @@ module ReductionMsg
       else if str == "False" then return false;
       throw new owned ErrorWithMsg("message: skipNan must be of type bool");
     }
-}
 
+    proc registerMe() {
+      use CommandMap;
+      registerFunction("segmentedReduction", segmentedReductionMsg);
+      registerFunction("reduction", reductionMsg);
+      registerFunction("countReduction", countReductionMsg);
+    }
+
+}

src/RegistrationMsg.chpl

@@ -135,4 +135,11 @@ module RegistrationMsg
         regLogger.debug(getModuleName(),getRoutineName(),getLineNumber(),repMsg);
         return new MsgTuple(repMsg, MsgType.NORMAL);
     }
+
+    proc registerMe() {
+      use CommandMap;
+      registerFunction("register", registerMsg);
+      registerFunction("attach", attachMsg);
+      registerFunction("unregister", unregisterMsg);
+    }
 }

src/SegmentedMsg.chpl

@@ -942,4 +942,22 @@ module SegmentedMsg {
       smLogger.debug(getModuleName(),getRoutineName(),getLineNumber(),repMsg);
       return new MsgTuple(repMsg, MsgType.NORMAL);
   }
-}
+
+  proc registerMe() {
+    use CommandMap;
+    registerFunction("segmentLengths", segmentLengthsMsg);
+    registerFunction("segmentedHash", segmentedHashMsg);
+    registerFunction("segmentedEfunc", segmentedEfuncMsg);
+    registerFunction("segmentedFindLoc", segmentedFindLocMsg);
+    registerFunction("segmentedFindAll", segmentedFindAllMsg);
+    registerFunction("segmentedPeel", segmentedPeelMsg);
+    registerFunction("segmentedSub", segmentedSubMsg);
+    registerFunction("segmentedIndex", segmentedIndexMsg);
+    registerFunction("segmentedBinopvv", segBinopvvMsg);
+    registerFunction("segmentedBinopvs", segBinopvsMsg);
+    registerFunction("segmentedGroup", segGroupMsg);
+    registerFunction("segmentedIn1d", segIn1dMsg);
+    registerFunction("randomStrings", randomStringsMsg);
+    registerFunction("segStr-assemble", assembleStringsMsg);
+  }
+}

src/SortMsg.chpl

@@ -104,4 +104,9 @@ module SortMsg
       sortLogger.debug(getModuleName(),getRoutineName(),getLineNumber(),repMsg);      
       return new MsgTuple(repMsg, MsgType.NORMAL);
     }// end sortMsg()
+
+    proc registerMe() {
+      use CommandMap;
+      registerFunction("sort", sortMsg);
+    }
 }// end module SortMsg

src/UniqueMsg.chpl

@@ -225,4 +225,10 @@ module UniqueMsg
         umLogger.debug(getModuleName(),getRoutineName(),getLineNumber(),repMsg);
         return new MsgTuple(repMsg, MsgType.NORMAL);
     }
+
+    proc registerMe() {
+      use CommandMap;
+      registerFunction("unique", uniqueMsg);
+      registerFunction("value_counts", value_countsMsg);
+    }
 }