Add unit tests for api-tools/parser/gradeLoader #44

build.bat

@@ -24,8 +24,8 @@ echo Performing checks...
 go mod tidy && ^
 go vet ./... && ^
 staticcheck ./... && ^
-gofmt -w ./.. && ^
-goimports -w ./..
+gofmt -w . && ^
+goimports -w .
 if ERRORLEVEL 1 exit /b %ERRORLEVEL% :: fail if error occurred
 echo Checks done!
 if %skip%==1 exit

parser/gradeLoader.go

@@ -5,79 +5,64 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
-)
 
-var grades = []string{"A+", "A", "A-", "B+", "B", "B-", "C+", "C", "C-", "D+", "D", "D-", "F", "W", "P", "CR", "NC", "I"}
+	"github.com/UTDNebula/api-tools/utils"
+)
 
-func loadGrades(csvDir string) map[string]map[string][]int {
+var (
+	grades          = []string{"A+", "A", "A-", "B+", "B", "B-", "C+", "C", "C-", "D+", "D", "D-", "F", "W", "P", "CR", "NC", "I"}
+	optionalColumns = []string{"W", "P", "CR", "NC", "I"}
+	requiredColumns = []string{"Section", "Subject", "Catalog Number", "A+"}
+	semesterRegex   = regexp.MustCompile(`[1-9][0-9][USF]`)
+)
 
+func loadGrades(csvDir string) (map[string]map[string][]int, error) {
 	// MAP[SEMESTER] -> MAP[SUBJECT + NUMBER + SECTION] -> GRADE DISTRIBUTION
 	gradeMap := make(map[string]map[string][]int)
 
-	if csvDir == "" {
-		log.Print("No grade data CSV directory specified. Grade data will not be included.")
-		return gradeMap
-	}
-
-	dirPtr, err := os.Open(csvDir)
-	if err != nil {
-		panic(err)
-	}
-	defer dirPtr.Close()
+	fileNames := utils.GetAllFilesWithExtension(csvDir, ".csv")
+	for _, name := range fileNames {
 
-	csvFiles, err := dirPtr.ReadDir(-1)
-	if err != nil {
-		panic(err)
-	}
-
-	for _, csvEntry := range csvFiles {
-
-		if csvEntry.IsDir() {
-			continue
+		semester := semesterRegex.FindString(name)
+		if semester == "" {
+			return gradeMap, fmt.Errorf("invalid name %s, must match format {>10}{F,S,U} i.e. 22F", name)
 		}
 
-		csvPath := fmt.Sprintf("%s/%s", csvDir, csvEntry.Name())
-
-		csvFile, err := os.Open(csvPath)
+		var err error
+		gradeMap[semester], err = csvToMap(name)
 		if err != nil {
-			panic(err)
-		}
-		defer csvFile.Close()
-
-		// Create logs directory
-		if _, err := os.Stat("./logs/grades"); err != nil {
-			os.Mkdir("./logs/grades", os.ModePerm)
+			return gradeMap, fmt.Errorf("error parsing %s: %v", name, err)
 		}
+	}
 
-		// Create log file [name of csv].log in logs directory
-		basePath := filepath.Base(csvPath)
-		csvName := strings.TrimSuffix(basePath, filepath.Ext(basePath))
-		logFile, err := os.Create("./logs/grades/" + csvName + ".log")
+	return gradeMap, nil
+}
 
-		if err != nil {
-			log.Panic("Could not create CSV log file.")
+func csvToMap(filename string) (map[string][]int, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, fmt.Errorf("error opening CSV file '%s': %v", filename, err)
+	}
+	defer func(file *os.File) {
+		if err := file.Close(); err != nil {
+			log.Printf("failed to close file '%s': %v", filename, err)
 		}
-		defer logFile.Close()
+	}(file)
 
-		// Put data from csv into map
-		gradeMap[csvName] = csvToMap(csvFile, logFile)
+	reader := csv.NewReader(file)
+	records, err := reader.ReadAll()
+	if err != nil {
+		return nil, fmt.Errorf("error parsing %s: %v", filename, err)
 	}
 
-	return gradeMap
-}
-
-func csvToMap(csvFile *os.File, logFile *os.File) map[string][]int {
-	reader := csv.NewReader(csvFile)
-	records, err := reader.ReadAll() // records is [][]strings
-	if err != nil {
-		log.Panicf("Error parsing %s: %s", csvFile.Name(), err.Error())
+	if len(records) == 0 {
+		return nil, fmt.Errorf("empty CSV file '%s'", filename)
 	}
 
 	indexMap := make(map[string]int)
-
 	for j, col := range records[0] {
 		switch col {
 		case "Catalog Number", "Catalog Nbr":
@@ -89,18 +74,15 @@ func csvToMap(csvFile *os.File, logFile *os.File) map[string][]int {
 		}
 	}
 
-	// required columns
-	for _, name := range []string{"Section", "Subject", "Catalog Number", "A+"} {
+	for _, name := range requiredColumns {
 		if _, ok := indexMap[name]; !ok {
-			fmt.Fprintf(logFile, "could not find %s column", name)
-			log.Panicf("could not find %s column", name)
+			return nil, fmt.Errorf("could not find %s column in %s", name, filename)
 		}
 	}
 
-	// optional columns
-	for _, name := range []string{"W", "P", "CR", "NC", "I"} {
+	for _, name := range optionalColumns {
 		if _, ok := indexMap[name]; !ok {
-			logFile.WriteString(fmt.Sprintf("could not find %s column\n", name))
+			log.Printf("could not find %s column in %s", name, filename)
 		}
 	}
 
@@ -109,7 +91,6 @@ func csvToMap(csvFile *os.File, logFile *os.File) map[string][]int {
 	catalogNumberCol := indexMap["Catalog Number"]
 
 	distroMap := make(map[string][]int)
-
 	for _, record := range records[1:] {
 		// convert grade distribution from string to int
 		intSlice := make([]int, len(grades))
@@ -125,5 +106,6 @@ func csvToMap(csvFile *os.File, logFile *os.File) map[string][]int {
 		distroKey := record[subjectCol] + record[catalogNumberCol] + trimmedSectionNumber
 		distroMap[distroKey] = intSlice[:]
 	}
-	return distroMap
+
+	return distroMap, nil
 }

parser/gradeLoader_test.go

@@ -0,0 +1,134 @@
+package parser
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+var (
+	gradeLoaderTestCases = map[string]struct {
+		csvContent string
+		want       map[string][]int
+		fail       bool
+	}{
+		"Valid_Data": {
+			csvContent: `Instructor 1,Instructor 2,Instructor 3,Instructor 4,Instructor 5,Instructor 6,Subject,"Catalog Nbr",Section,A+,A,A-,B+,B,B-,C+,C,C-,D+,D,D-,F,NF,CR,I,NC,P,W
+"Curchack, Fred",,,,,,AP,3300,501,6,4,2,2,1,3,1,1,,,,,1,,,,,,0
+"Anjum, Zafar",,,,,,ARAB,1311,001,,26,,,1,,,,,,,,,,,,,,2`,
+			want: map[string][]int{
+				"AP3300501": {6, 4, 2, 2, 1, 3, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0},
+				"ARAB13111": {0, 26, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0},
+			},
+			fail: false,
+		},
+		"Missing_Required_Column_A+": {
+			csvContent: `Subject,"Catalog Nbr",Section,A,A-,B+
+CS,1337,001,10,5,5`,
+			fail: true,
+		},
+		"Missing_Required_Column_Subject": {
+			csvContent: `Instructor,"Catalog Nbr",Section,A+,A
+Doe,1337,001,10,5`,
+			fail: true,
+		},
+		"Empty_File": {
+			csvContent: ``,
+			fail:       true,
+		},
+	}
+)
+
+func TestLoadGrades(t *testing.T) {
+
+	invalidCSVNames := []string{"22", "2F", "2022F", "20-U", "15Fall"}
+
+	for i, name := range invalidCSVNames {
+		t.Run(
+			fmt.Sprintf("Invalid_CSV_Name_%d", i), func(t *testing.T) {
+				tempDir := t.TempDir()
+
+				temp, err := os.Create(filepath.Join(tempDir, name+".csv"))
+				if err != nil {
+					t.Errorf("failed to create temp file: %v", err)
+				}
+				defer temp.Close()
+
+				_, err = loadGrades(tempDir)
+				if err == nil {
+					t.Errorf("expected error but got none")
+				}
+			},
+		)
+	}
+
+	validCSVNames := []string{"25F", "18U", "26S"}
+	for i, name := range validCSVNames {
+		t.Run(
+			fmt.Sprintf("Valid_CSV_Name_%d", i), func(t *testing.T) {
+				tempDir := t.TempDir()
+
+				temp, err := os.Create(filepath.Join(tempDir, name+".csv"))
+				if err != nil {
+					t.Errorf("failed to create temp file: %v", err)
+				}
+				defer temp.Close()
+
+				_, err = temp.WriteString(gradeLoaderTestCases["Valid_Data"].csvContent)
+				if err != nil {
+					t.Errorf("failed to write test data: %v", err)
+				}
+
+				_, err = loadGrades(tempDir)
+				if err != nil {
+					t.Errorf("valid .csv failed: %v", err)
+				}
+			},
+		)
+	}
+
+	t.Run("Real_Data", func(t *testing.T) {
+		_, err := loadGrades("../grade-data/")
+		if err != nil {
+			t.Errorf("failed to load grades: %v", err)
+		}
+	})
+}
+
+func TestCSVToMap(t *testing.T) {
+	tempDir := t.TempDir()
+
+	for name, testCase := range gradeLoaderTestCases {
+		t.Run(name, func(t *testing.T) {
+
+			temp, err := os.CreateTemp(tempDir, "grades*.csv")
+			if err != nil {
+				t.Errorf("failed to create temp file: %v", err)
+			}
+			defer temp.Close()
+
+			if _, err = temp.WriteString(testCase.csvContent); err != nil {
+				t.Errorf("failed to write test data: %v", err)
+			}
+
+			output, err := csvToMap(temp.Name())
+			if err != nil {
+				if testCase.fail {
+					return
+				}
+				t.Errorf("failed to load csv: %v", err)
+			} else if testCase.fail {
+				t.Errorf("expected failure but got none")
+			} else {
+				diff := cmp.Diff(testCase.want, output)
+				if diff != "" {
+					t.Errorf("Failed (-expected +got)\n %s", diff)
+				}
+			}
+
+		})
+	}
+}

parser/parser.go

@@ -40,17 +40,25 @@ var (
 	timeLocation, timeError = time.LoadLocation("America/Chicago")
 )
 
+func init() {
+	if timeError != nil {
+		log.Fatalf("Failed to initialize timeLocation: %v", timeError)
+	}
+}
+
 // Parse loads scraped course artifacts, applies parsing and validation, and persists structured results.
 func Parse(inDir string, outDir string, csvPath string, skipValidation bool) {
 
-	// Panic if timeLocation didn't load properly
-	if timeError != nil {
-		panic(timeError)
-	}
+	if csvPath == "" {
+		log.Print("No grade data CSV directory specified. Grade data will not be included.")
+	} else {
+		var err error
+		GradeMap, err = loadGrades(csvPath)
 
-	// Load grade data from csv in advance
-	GradeMap = loadGrades(csvPath)
-	if len(GradeMap) != 0 {
+		if err != nil {
+			log.Fatalf("Failed to load grade data: %v", err)
+			return
+		}
 		log.Printf("Loaded grade distributions for %d semesters.", len(GradeMap))
 	}