diff --git a/README.NotOnlyFLOPs.md b/README.NotOnlyFLOPs.md
new file mode 100644
index 00000000..18841c66
--- /dev/null
+++ b/README.NotOnlyFLOPs.md
@@ -0,0 +1,49 @@
+# What have we changed?
+
+In the modifications of the code, we have added the required functions and variables that we have needed to implement the Task 3 and Task 4 of the Coding Challenge.
+
+We have tried to follow the basis of the profiler structure, in order to follow an easy integration and update of future codes.
+
+We have also followed the way of generating the diagrams, using gnuplot, following the same pattern of generating gnuplots as the current profiler.
+
+# Changes made for Task 3
+
+## Generating heatmaps
+
+For the creation of the heatmaps for the different patterns, in the function `Analyze` of the `profiler.go` file we have added some function calls at STEP 6.
+
+- The function `GetSendDataForTask3` gets the array of patterns of bytes sent from the sender ranks to the receiver ranks. These data are obtained from the `counts_reader.go` file, inside the function `LoadCallsData`, where we append the `sendCounts` map to the array of patterns in each iteration. If we have more than one communicator, this array will have all patterns of all communicators.
+
+- As we only want the first patterns of the first communicator, with the function GetNumberSendDataForTask3(), where we get the number of the patterns in one communicator. Next we create an slice with only the patterns we want.
+
+- With the function `GetNumberOfCalls` we get an array with the proportion of pattern calls over the total number of calls. To get it, inside the function we read the file `send-counters.job0.rank0.txt`, and we keep only the strings with the proportions (e.g., 121/964).
+
+- Finally we have the call `Task3`, which generates the plots using the returns of the two previous functions.
+This function, located at `plot.go`, iterates over the patterns, and in every iteration generates a matrix ready to be plotted, using the original matrix of bytes sent between ranks.
+At the end of every iteration, generates the gnuplot.
+
+## Visualization in the WebUI
+
+To show the gnuplots in the WebUI, we have added a tab called Task3, with the same layout as the Calls tab.
+
+We have only edited the files `webui.go` and `index.html`, and added the files `task3Details.html` and `task3Layout`.
+
+Inside the files we have "replicated" the functions and variables of the call and calls data. The major difference is in the function `serviceTask3HeatmapDetailsRequest`, where we only watch for the param that indicates the number of pattern.
+
+The two added files `task3Details.html` and `task3Layout` are also very similar to the `callDetails.html` and `callsLayout` files.
+
+The table showing the number of patterns adapts the size with the number of patterns found, with a maximum of 10 (the 10 heaviest patterns).
+
+# Changes made for Task 4
+
+## Generating heatmaps
+
+To generate the weighted sum of all patterns, we have added a call to `Task4` in the file `profiler.go`, following the `Task3` call.
+
+All the steps are almost the same as the ones we have done for task 3.
+
+Inside `Task4` function we first calculate the sum of all pattern cells for every cell, and then we proceed to generate the plot.
+
+## Visualization in the WebUI
+
+For the visualization of the generated gnuplot in the WebUI we have followed the same steps as for the task3, although now we only show one plot in the `Task4` tab.
diff --git a/tools/internal/pkg/counts/counts.go b/tools/internal/pkg/counts/counts.go
index 1d908ac7..6255179e 100644
--- a/tools/internal/pkg/counts/counts.go
+++ b/tools/internal/pkg/counts/counts.go
@@ -89,6 +89,8 @@ type CallData struct {
// RecvData is all the data from the receive counts
RecvData Data
+
+ SccPattern map[int][]int
}
// Stats represent the stats related to counts of a specific collective operation
@@ -696,3 +698,60 @@ func GatherStatsFromCallData(cd map[int]*CallData, sizeThreshold int) (SendRecvS
return cs, nil
}
+
+// GetFilePath returns the full path to the pattern file associated to a rank within a job
+func GetFilePath(basedir string, jobid int, rank int) string {
+ return filepath.Join(basedir, fmt.Sprintf("send-counters.job%d.rank%d.txt", jobid, rank))
+}
+
+// Returns an slice with the proportion of calls for pattern, e.g. 361/964 calls
+func GetNumberOfCalls(dir string, jobid int, callNum int) ([]string, error) {
+ var numberCallsCleaned []string
+ var numberTotalCallsCleaned []string
+
+ // Prepare the file to read
+ countsOutputFile := GetFilePath(dir, jobid, callNum)
+ countsFd, err := os.Open(countsOutputFile)
+ if err != nil {
+ return numberCallsCleaned, err
+ }
+ defer countsFd.Close()
+ countsReader := bufio.NewReader(countsFd)
+
+ // The very first line should be '#Raw counters'
+ line, readerErr := countsReader.ReadString('\n')
+ if readerErr != nil {
+ return numberCallsCleaned, readerErr
+ }
+ if line != "# Raw counters\n" {
+ return numberCallsCleaned, fmt.Errorf("wrong file format: %s", line)
+ }
+
+ // Read the file until EOF. In each iteration we only get the string XXX/YYY from the file and append to numberCallsCleaned
+ for {
+ line, readerErr := countsReader.ReadString('\n')
+ if readerErr != nil && readerErr != io.EOF {
+ return numberCallsCleaned, readerErr
+ }
+ if readerErr == io.EOF {
+ break
+ }
+
+ if strings.HasPrefix(line, "Alltoallv calls ") {
+ numberTotalCalls := strings.Split(line, "-")
+ numberTotalCallsCleaned_aux := strings.TrimSuffix(numberTotalCalls[1], "\n")
+ numberTotalCallsCleaned = append(numberTotalCallsCleaned, numberTotalCallsCleaned_aux)
+ }
+
+ if strings.HasPrefix(line, "Count: ") {
+ numberCalls := strings.Split(line, " ")
+ numberCallsCleaned = append(numberCallsCleaned, numberCalls[1])
+ }
+ }
+ for i := 0; i < len(numberCallsCleaned); i++ {
+ total, _ := strconv.Atoi(numberTotalCallsCleaned[i])
+ numberCallsCleaned[i] = fmt.Sprintf("%s/%d", numberCallsCleaned[i], total+1)
+ }
+
+ return numberCallsCleaned, nil
+}
diff --git a/tools/internal/pkg/counts/counts_reader.go b/tools/internal/pkg/counts/counts_reader.go
index ea675943..00de828f 100644
--- a/tools/internal/pkg/counts/counts_reader.go
+++ b/tools/internal/pkg/counts/counts_reader.go
@@ -30,6 +30,10 @@ const (
rawCountsRecvCountsPrefix = "Recv counts"
)
+// Global variable for task 3. The outer slice is for the number of the pattern. The map is a matrix of sender->receiver
+var SendDataForTask3 []map[int][]int
+var NumberSendDataForTask3 int
+
// AnalyzeCounts analyses the count from a count file
func AnalyzeCounts(counts []string, msgSizeThreshold int, datatypeSize int) (Stats, map[int][]int, error) {
var stats Stats
@@ -452,6 +456,8 @@ func LoadCallsData(sendCountsFile, recvCountsFile string, rank int, msgSizeThres
}
defer sendFile.Close()
reader := bufio.NewReader(sendFile)
+
+ NumberSendDataForTask3 = 0
for {
cd := new(CallData)
cd.SendData.CountsMetadata, readerErr = GetHeader(reader)
@@ -484,6 +490,10 @@ func LoadCallsData(sendCountsFile, recvCountsFile string, rank int, msgSizeThres
cd.SendData.Counts[callID] = sendCounts
}
+ // Append the counts found on the array for data send
+ NumberSendDataForTask3++
+ SendDataForTask3 = append(SendDataForTask3, sendCounts)
+
if readerErr == io.EOF {
break
}
diff --git a/tools/internal/pkg/patterns/patterns.go b/tools/internal/pkg/patterns/patterns.go
index f8fd2ea4..6ae97681 100644
--- a/tools/internal/pkg/patterns/patterns.go
+++ b/tools/internal/pkg/patterns/patterns.go
@@ -607,3 +607,12 @@ func WriteData(patternsFd *os.File, patternsSummaryFd *os.File, patternsData Dat
return nil
}
+
+// Returns the array of maps of the send counts. The outer array is the number of pattern
+func GetSendDataForTask3() ([]map[int][]int) {
+ return counts.SendDataForTask3;
+}
+
+func GetNumberSendDataForTask3() (int) {
+ return counts.NumberSendDataForTask3;
+}
diff --git a/tools/internal/pkg/plot/plot.go b/tools/internal/pkg/plot/plot.go
index 8d54a038..396538bf 100644
--- a/tools/internal/pkg/plot/plot.go
+++ b/tools/internal/pkg/plot/plot.go
@@ -703,3 +703,237 @@ func Avgs(dir string, outputDir string, numRanks int, hostMap map[string][]int,
return runGnuplot(gnuplotScript, outputDir)
}
+
+// Write into a file the "body" of the gnuplot
+func writeHeatmaps(fd *os.File, HeatMatrix [][]int, outputDir string, numRanks int) error {
+
+ // Matrix to plot
+ str := "$map2 << EOD\n"
+ for i := numRanks-1; i >= 0; i-- {
+ for j := 0; j < numRanks; j++ {
+ str += fmt.Sprintf(" %d", HeatMatrix[i][j])
+ }
+ str += "\n"
+ }
+
+ str += "EOD\n\n"
+
+ // Extra information to display the plot correctly
+ str += "set rmargin 10\n"
+ str += "set yrange [-0.5:*]\n"
+ str += "set xrange [-0.5:*]\n"
+ str += "set cbrange [0:7]\n"
+ str += "set xlabel \"Senders\"\n"
+ str += "set ylabel \"Receivers\"\n"
+ if numRanks <= 40 {
+ str += "set xtics 1\n"
+ str += "set ytics 1\n"
+ } else {
+ str += "set xtics rotate by 22.5\n"
+ str += "set xtics 5 offset 0,-0.75,0\n"
+ str += "set ytics 5\n"
+ }
+ str += "set cbtics (\"0\" 0,\"1-10\" 1,\"11-100\" 2,\"101-1,000\" 3,\"1,001-10,000\" 4,\"10,001-100,000\" 5,\"100,001-1,000,000\" 6,\"1,000,001-max\" 7)\n"
+ str += "set palette defined (0 \"white\", 0.5 \"white\", 0.5 \"yellow\", 1.5 \"yellow\", 1.5 \"orange\", 2.5 \"orange\", 2.5 \"green\", 3.5 \"green\", 3.5 \"red\", 4.5 \"red\", 4.5 \"purple\", 5.5 \"purple\", 5.5 \"brown\", 6.5 \"brown\", 6.5 \"black\", 7 \"black\")\n"
+ str += "unset key\n"
+ str += "plot '$map2' using 1:2:3 matrix with image\n"
+
+ _, err := fd.WriteString(str)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// Creates the gnuplot script of a pattern
+func generateTask3Plots(outputDir string, HeatMatrix [][]int, numRanks, numPattern int, NumberOfCalls string) (string, error) {
+
+ namePlot := fmt.Sprintf("heatmap-task3-pattern%d.gnuplot", numPattern)
+ plotScriptFile := filepath.Join(outputDir, namePlot)
+ fd, err := os.OpenFile(plotScriptFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
+ if err != nil {
+ return "", err
+ }
+ defer fd.Close()
+
+ // Gnuplot "header"
+ str := "set term png size 1200,900\n"
+ str += fmt.Sprintf("set output \"heatmap-task3-pattern%d.png\"\n\n", numPattern)
+ str += fmt.Sprintf("set title \"Heatmap of Pattern %d: %s calls.\"\n\n", numPattern, NumberOfCalls)
+
+ _, err = fd.WriteString(str)
+ if err != nil {
+ return "", err
+ }
+ // Gnuplot "body"
+ err = writeHeatmaps(fd, HeatMatrix, outputDir, numRanks)
+ if err != nil {
+ return "", err
+ }
+
+ return plotScriptFile, nil
+}
+
+// Generation of heatmaps plots of Task 3
+func Task3(outputDir string, SendDataForTask3 []map[int][]int, NumberOfCalls []string) error {
+
+ // Total of patterns
+ var numPatterns = len(SendDataForTask3)
+
+ // If there are no patterns, nothing is displayed
+ if numPatterns == 0 {
+ return nil
+ }
+
+ // Total of ranks of the pattern calls
+ var numRanks = len(SendDataForTask3[0])
+
+ // For every pattern, we map the info of the original matrix to a matrix to print with gnuplot
+ for i := 0; i < numPatterns; i++ {
+ // Allocate and initialize matrix to put inside the plot
+ HeatMatrix := make([][]int, numRanks)
+ for ii := 0; ii < numRanks; ii++ {
+ HeatMatrix[ii] = make([]int, numRanks)
+ }
+ // Prepare matrix with mapped values
+ for j := 0; j < numRanks; j++ {
+ for k := 0; k < numRanks; k++ {
+ OriginalValue := SendDataForTask3[numPatterns-1-i][j][k]
+ NewValue := 0
+ switch {
+ case OriginalValue == 0:
+ NewValue = 0
+ case 0 < OriginalValue && OriginalValue <= 10:
+ NewValue = 1
+ case 10 < OriginalValue && OriginalValue <= 100:
+ NewValue = 2
+ case 100 < OriginalValue && OriginalValue <= 1000:
+ NewValue = 3
+ case 1000 < OriginalValue && OriginalValue <= 10000:
+ NewValue = 4
+ case 10000 < OriginalValue && OriginalValue <= 100000:
+ NewValue = 5
+ case 100000 < OriginalValue && OriginalValue <= 1000000:
+ NewValue = 6
+ case 1000000 < OriginalValue:
+ NewValue = 7
+ }
+
+ // We start putting values from the bottom row to generate the plot correctly
+ HeatMatrix[numRanks-1-k][j] = NewValue
+ }
+ }
+ // Generate the plot script with all the previous info
+ gnuplotScript, err := generateTask3Plots(outputDir, HeatMatrix, numRanks, i, NumberOfCalls[numPatterns-1-i])
+ if err != nil {
+ return fmt.Errorf("generateTask3Plots() failed: %s", err)
+ }
+ runGnuplot(gnuplotScript, outputDir)
+ }
+
+ return nil
+}
+
+// Creates the gnuplot script of a pattern for task4
+func generateTask4Plots(outputDir string, HeatMatrix [][]int, numRanks int) (string, error) {
+
+ namePlot := fmt.Sprintf("heatmap-task4-allpatterns.gnuplot")
+ plotScriptFile := filepath.Join(outputDir, namePlot)
+ fd, err := os.OpenFile(plotScriptFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
+ if err != nil {
+ return "", err
+ }
+ defer fd.Close()
+
+ // Gnuplot "header"
+ str := "set term png size 1200,900\n"
+ str += fmt.Sprintf("set output \"heatmap-task4-allpatterns.png\"\n\n")
+ str += fmt.Sprintf("set title \"Heatmap of the sum of all patterns.\"\n\n")
+
+ _, err = fd.WriteString(str)
+ if err != nil {
+ return "", err
+ }
+ // Gnuplot "body"
+ err = writeHeatmaps(fd, HeatMatrix, outputDir, numRanks)
+ if err != nil {
+ return "", err
+ }
+
+ return plotScriptFile, nil
+}
+
+// Generation of the heatmap plot of Task 4
+func Task4(outputDir string, SendDataForTask3 []map[int][]int, NumberOfCalls []string) error {
+
+ // Total of patterns
+ var numPatterns = len(SendDataForTask3)
+
+ // If there are no patterns, nothing is displayed
+ if numPatterns == 0 {
+ return nil
+ }
+
+ // Total of ranks of the pattern calls
+ var numRanks = len(SendDataForTask3[0])
+
+ // Allocate and initialize matrix to put inside the plot
+ HeatMatrix := make([][]int, numRanks)
+ HeatMatrixAux := make([][]float64, numRanks)
+ for ii := 0; ii < numRanks; ii++ {
+ HeatMatrix[ii] = make([]int, numRanks)
+ HeatMatrixAux[ii] = make([]float64, numRanks)
+ }
+
+ // Calcule the weighted sum of the patterns
+ for i := 0; i < numPatterns; i++ {
+ weightSplitted := strings.Split(NumberOfCalls[numPatterns-1-i], "/")
+ weight, _ := strconv.Atoi(weightSplitted[0])
+ totalWeight, _ := strconv.Atoi(weightSplitted[1])
+ // Prepare matrix with mapped values
+ for j := 0; j < numRanks; j++ {
+ for k := 0; k < numRanks; k++ {
+ OriginalValue := SendDataForTask3[numPatterns-1-i][j][k]
+ // We start putting values from the bottom row to generate the plot correctly
+ HeatMatrixAux[numRanks-1-k][j] += float64(OriginalValue)*(float64(weight)/float64(totalWeight))
+ }
+ }
+ }
+
+ // Prepare matrix with mapped values
+ for j := 0; j < numRanks; j++ {
+ for k := 0; k < numRanks; k++ {
+ OriginalValue := HeatMatrixAux[j][k]
+ NewValue := 0
+ switch {
+ case OriginalValue == 0:
+ NewValue = 0
+ case 0 < OriginalValue && OriginalValue <= 10:
+ NewValue = 1
+ case 10 < OriginalValue && OriginalValue <= 100:
+ NewValue = 2
+ case 100 < OriginalValue && OriginalValue <= 1000:
+ NewValue = 3
+ case 1000 < OriginalValue && OriginalValue <= 10000:
+ NewValue = 4
+ case 10000 < OriginalValue && OriginalValue <= 100000:
+ NewValue = 5
+ case 100000 < OriginalValue && OriginalValue <= 1000000:
+ NewValue = 6
+ case 1000000 < OriginalValue:
+ NewValue = 7
+ }
+ HeatMatrix[k][j] = NewValue
+ }
+ }
+
+ // Generate the plot script with all the previous info
+ gnuplotScript, err := generateTask4Plots(outputDir, HeatMatrix, numRanks)
+ if err != nil {
+ return fmt.Errorf("generateTask3Plots() failed: %s", err)
+ }
+ runGnuplot(gnuplotScript, outputDir)
+
+ return nil
+}
diff --git a/tools/internal/pkg/profiler/profiler.go b/tools/internal/pkg/profiler/profiler.go
index 7f012be7..2795e6f7 100644
--- a/tools/internal/pkg/profiler/profiler.go
+++ b/tools/internal/pkg/profiler/profiler.go
@@ -1118,6 +1118,35 @@ func (cfg *PostmortemConfig) Analyze() error {
if err != nil {
return fmt.Errorf("unable to plot average data: %s", err)
}
+
+ // Heatmaps of task 3
+ // Get the array of maps of the send counts. The lenght take into account all the comunicators
+ SendDataForTask3 := patterns.GetSendDataForTask3()
+ // Get the number of patterns per comunicator
+ NumberSendDataForTask3 := patterns.GetNumberSendDataForTask3()
+
+ // We get only the number of patterns of the first comunicator
+ var PatternsFound []map[int][]int
+ for i := 0; i < NumberSendDataForTask3; i++ {
+ PatternsFound = append(PatternsFound, SendDataForTask3[i])
+ }
+ // Get an array with the proportion of calls of the pattern over the total number of calls
+ NumberOfCalls, err := counts.GetNumberOfCalls(cfg.DatasetDir, 0, 0)
+ if err != nil {
+ return fmt.Errorf("unable to GetNumberOfCalls: %s", err)
+ }
+ // Creation of the plots
+ err = plot.Task3(cfg.DatasetDir, PatternsFound, NumberOfCalls)
+ if err != nil {
+ return fmt.Errorf("unable to plot heatmap of Task3: %s", err)
+ }
+
+ // Heatmap of task 4
+ err = plot.Task4(cfg.DatasetDir, PatternsFound, NumberOfCalls)
+ if err != nil {
+ return fmt.Errorf("unable to plot heatmap of Task4: %s", err)
+ }
+
duration := t.Stop()
fmt.Printf("Step completed in %s\n", duration)
} else {
diff --git a/tools/internal/pkg/webui/templates/index.html b/tools/internal/pkg/webui/templates/index.html
index e1897e42..020c2dda 100644
--- a/tools/internal/pkg/webui/templates/index.html
+++ b/tools/internal/pkg/webui/templates/index.html
@@ -51,6 +51,8 @@
+
+
@@ -60,6 +62,14 @@
+
+
+
+
+
+
+
+
-
\ No newline at end of file
+
diff --git a/tools/internal/pkg/webui/templates/task3Details.html b/tools/internal/pkg/webui/templates/task3Details.html
new file mode 100644
index 00000000..0421756e
--- /dev/null
+++ b/tools/internal/pkg/webui/templates/task3Details.html
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tools/internal/pkg/webui/templates/task3Layout.html b/tools/internal/pkg/webui/templates/task3Layout.html
new file mode 100644
index 00000000..e39b9e15
--- /dev/null
+++ b/tools/internal/pkg/webui/templates/task3Layout.html
@@ -0,0 +1,115 @@
+
+
+
+
++
+
+
+
+