diff --git a/pom.xml b/pom.xml index 9d537518..ff52d4b8 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,21 @@ Java.net repository http://download.java.net/maven/2 + + bedatadriven + bedatadriven public repo + https://nexus.bedatadriven.com/content/groups/public/ + + + sonatype-snapshots + https://oss.sonatype.org/content/repositories/snapshots/ + + false + + + true + + @@ -230,6 +245,22 @@ xchart 3.6.3 + + org.renjin + renjin-script-engine + 3.5-beta43 + + + org.renjin.cran + pscl + 1.5.2-b14 + + + com.github.fommil.netlib + all + 1.1.2 + pom + UTF-8 diff --git a/src/main/java/de/tum/bgu/msm/data/DataSet.java b/src/main/java/de/tum/bgu/msm/data/DataSet.java index 027e46b0..745f2314 100644 --- a/src/main/java/de/tum/bgu/msm/data/DataSet.java +++ b/src/main/java/de/tum/bgu/msm/data/DataSet.java @@ -7,8 +7,11 @@ import de.tum.bgu.msm.data.travelTimes.TravelTimes; import de.tum.bgu.msm.modules.modeChoice.ModeChoiceCalibrationData; import org.matsim.api.core.v01.population.Population; +import org.renjin.primitives.vector.RowNamesVector; +import org.renjin.sexp.*; import java.util.*; +import java.util.function.Predicate; public class DataSet { @@ -186,6 +189,22 @@ public static int getStudentsForHousehold(MitoHousehold household) { } + public static int countMembersByAttribute(MitoHousehold household,int minAge,int maxAge) { + return (int) household.getPersons().values().stream().filter(person -> + person.getAge() >= minAge & person.getAge() <= maxAge).count(); + } + + public static int countMembersByAttribute(MitoHousehold household,int minAge,int maxAge, MitoOccupationStatus occupation) { + return (int) household.getPersons().values().stream().filter(person -> + person.getAge() >= minAge & person.getAge() <= maxAge & + person.getMitoOccupationStatus().equals(occupation)).count(); + } + + public static int countMembersByFilter(MitoHousehold household, Predicate filter) { + return (int) household.getPersons().values().stream().filter(filter).count(); + } + + public static int getLicenseHoldersForHousehold(MitoHousehold household) { return (int) household.getPersons().values().stream().filter(MitoPerson::hasDriversLicense).count(); } @@ -242,6 +261,195 @@ public ModeChoiceCalibrationData getModeChoiceCalibrationData() { return modeChoiceCalibrationData; } + private int determineAreaType(MitoHousehold hh) { + int areaType = -1; + if (hh.getHomeZone() != null) { + areaType = hh.getHomeZone().getAreaTypeSG().code() / 10; + } else { + System.out.println("Home MitoZone for Household " + hh.getId() + " is null!"); + } + return areaType; + } + + private ListVector RDataFrame; + public ListVector getRdataFrame() {return this.RDataFrame; } + + // Method to create R data frame + public void buildRdataFrame() { + + //Create Builders for each Variable + IntArrayVector.Builder hhId = new IntArrayVector.Builder(); + + IntArrayVector.Builder hhSize = new IntArrayVector.Builder(); + IntArrayVector.Builder hhSize1 = new IntArrayVector.Builder(); + IntArrayVector.Builder hhSize2 = new IntArrayVector.Builder(); + IntArrayVector.Builder hhSize3 = new IntArrayVector.Builder(); + IntArrayVector.Builder hhSize23 = new IntArrayVector.Builder(); + IntArrayVector.Builder hhSize4 = new IntArrayVector.Builder(); + IntArrayVector.Builder hhSize5 = new IntArrayVector.Builder(); + IntArrayVector.Builder hhSize345 = new IntArrayVector.Builder(); + IntArrayVector.Builder hhSize45 = new IntArrayVector.Builder(); + + IntArrayVector.Builder hhPersons0to6 = new IntArrayVector.Builder(); + IntArrayVector.Builder hhPersons6to17 = new IntArrayVector.Builder(); + IntArrayVector.Builder hhPersons18to29_worker = new IntArrayVector.Builder(); + IntArrayVector.Builder hhPersons18to29_student = new IntArrayVector.Builder(); + IntArrayVector.Builder hhPersons18to29_unemployed = new IntArrayVector.Builder(); + IntArrayVector.Builder hhPersons30to64_worker = new IntArrayVector.Builder(); + IntArrayVector.Builder hhPersons30to64_student = new IntArrayVector.Builder(); + IntArrayVector.Builder hhPersons18to64_student = new IntArrayVector.Builder(); + IntArrayVector.Builder hhPersons30to64_unemployed = new IntArrayVector.Builder(); + IntArrayVector.Builder hhPersons18to64_unemployed = new IntArrayVector.Builder(); + IntArrayVector.Builder hhPersons65up = new IntArrayVector.Builder(); + + IntArrayVector.Builder hhPersonsFemale = new IntArrayVector.Builder(); + IntArrayVector.Builder hhPersonsWithMobilityRestriction = new IntArrayVector.Builder(); + + IntArrayVector.Builder hhEconomicStatus = new IntArrayVector.Builder(); + IntArrayVector.Builder hhEconomicStatus2 = new IntArrayVector.Builder(); + IntArrayVector.Builder hhEconomicStatus3 = new IntArrayVector.Builder(); + IntArrayVector.Builder hhEconomicStatus23 = new IntArrayVector.Builder(); + IntArrayVector.Builder hhEconomicStatus4 = new IntArrayVector.Builder(); + IntArrayVector.Builder hhEconomicStatus5 = new IntArrayVector.Builder(); + IntArrayVector.Builder hhEconomicStatus45 = new IntArrayVector.Builder(); + + IntArrayVector.Builder hhAutos = new IntArrayVector.Builder(); + DoubleArrayVector.Builder hhPropAutos = new DoubleArrayVector.Builder(); + + IntArrayVector.Builder hhRegionType = new IntArrayVector.Builder(); + IntArrayVector.Builder hhRegionType2 = new IntArrayVector.Builder(); + IntArrayVector.Builder hhRegionType3 = new IntArrayVector.Builder(); + IntArrayVector.Builder hhRegionType4 = new IntArrayVector.Builder(); + + // Loop through dataSet and assign values + final Iterator iterator = this.getHouseholds().values().iterator(); + for (; iterator.hasNext(); ) { + MitoHousehold hh = iterator.next(); + + hhId.add(hh.getId()); + int householdSize = hh.getHhSize(); + hhSize.add(householdSize); + hhSize1.add(householdSize == 1 ? 1 : 0); + hhSize2.add(householdSize == 2 ? 1 : 0); + hhSize23.add(householdSize == 2 | householdSize == 3 ? 1 : 0); + hhSize3.add(householdSize == 3 ? 1 : 0); + hhSize4.add(householdSize == 4 ? 1 : 0); + hhSize5.add(householdSize >= 5 ? 1 : 0); + hhSize345.add(householdSize >= 3 ? 1 : 0); + hhSize45.add(householdSize >= 4 ? 1 : 0); + + hhPersons0to6.add(DataSet.countMembersByFilter(hh, mitoPerson -> + mitoPerson.getAge() < 6)); + hhPersons6to17.add(DataSet.countMembersByFilter(hh, mitoPerson -> + mitoPerson.getAge() >= 6 & mitoPerson.getAge() <= 17)); + hhPersons18to29_worker.add(DataSet.countMembersByFilter(hh, mitoPerson -> + mitoPerson.getAge() >= 18 & mitoPerson.getAge() <= 29 & + mitoPerson.getMitoOccupationStatus().equals(MitoOccupationStatus.WORKER))); + hhPersons18to29_student.add(DataSet.countMembersByFilter(hh, mitoPerson -> + mitoPerson.getAge() >= 18 & mitoPerson.getAge() <= 29 & + mitoPerson.getMitoOccupationStatus().equals(MitoOccupationStatus.STUDENT))); + hhPersons18to29_unemployed.add(DataSet.countMembersByFilter(hh, mitoPerson -> + mitoPerson.getAge() >= 18 & mitoPerson.getAge() <= 29 & + mitoPerson.getMitoOccupationStatus().equals(MitoOccupationStatus.UNEMPLOYED))); + hhPersons30to64_worker.add(DataSet.countMembersByFilter(hh, mitoPerson -> + mitoPerson.getAge() >= 30 & mitoPerson.getAge() <= 64 & + mitoPerson.getMitoOccupationStatus().equals(MitoOccupationStatus.WORKER))); + hhPersons30to64_student.add(DataSet.countMembersByFilter(hh, mitoPerson -> + mitoPerson.getAge() >= 30 & mitoPerson.getAge() <= 64 & + mitoPerson.getMitoOccupationStatus().equals(MitoOccupationStatus.STUDENT))); + hhPersons18to64_student.add(DataSet.countMembersByFilter(hh, mitoPerson -> + mitoPerson.getAge() >= 18 & mitoPerson.getAge() <= 64 & + mitoPerson.getMitoOccupationStatus().equals(MitoOccupationStatus.STUDENT))); + hhPersons30to64_unemployed.add(DataSet.countMembersByFilter(hh, mitoPerson -> + mitoPerson.getAge() >= 30 & mitoPerson.getAge() <= 64 & + mitoPerson.getMitoOccupationStatus().equals(MitoOccupationStatus.UNEMPLOYED))); + hhPersons18to64_unemployed.add(DataSet.countMembersByFilter(hh, mitoPerson -> + mitoPerson.getAge() >= 18 & mitoPerson.getAge() <= 64 & + mitoPerson.getMitoOccupationStatus().equals(MitoOccupationStatus.UNEMPLOYED))); + hhPersons65up.add(DataSet.countMembersByFilter(hh, mitoPerson -> + mitoPerson.getAge() >= 65)); + + + hhPersonsFemale.add(DataSet.countMembersByFilter(hh, mitoPerson -> + mitoPerson.getMitoGender().equals(MitoGender.FEMALE))); + hhPersonsWithMobilityRestriction.add(0); //todo: include actual disability status (currently 0) + + int economicStatus = hh.getEconomicStatus(); + hhEconomicStatus.add(economicStatus); + hhEconomicStatus2.add(economicStatus == 2 ? 1 : 0); + hhEconomicStatus3.add(economicStatus == 3 ? 1 : 0); + hhEconomicStatus23.add(economicStatus == 2 || economicStatus == 3 ? 1 : 0); + hhEconomicStatus4.add(economicStatus == 4 ? 1 : 0); + hhEconomicStatus5.add(economicStatus == 5 ? 1 : 0); + hhEconomicStatus45.add(economicStatus >= 4 ? 1 : 0); + + hhAutos.add(hh.getAutos()); + hhPropAutos.add(Math.min(1.0,((double) hh.getAutos() / DataSet.countMembersByFilter(hh, mitoPerson -> + mitoPerson.getAge() >= 15)))); + + int hhAreaType = determineAreaType(hh); + hhRegionType.add(hhAreaType); + hhRegionType2.add(hhAreaType == 2 ? 1 : 0); + hhRegionType3.add(hhAreaType == 3 ? 1 : 0); + hhRegionType4.add(hhAreaType == 4 ? 1 : 0); + } + + // Add all vectors to a R data frame + ListVector.NamedBuilder RModelDataBuilder = new ListVector.NamedBuilder(); + RModelDataBuilder.setAttribute(Symbols.CLASS, StringVector.valueOf("data.frame")); + RModelDataBuilder.setAttribute(Symbols.ROW_NAMES, new RowNamesVector(this.getHouseholds().size())); + RModelDataBuilder.add("hh.id",hhId.build()); + RModelDataBuilder.add("hh.size",hhSize.build()); + RModelDataBuilder.add("hh.size_1",hhSize1.build()); + RModelDataBuilder.add("hh.size_2",hhSize2.build()); + RModelDataBuilder.add("hh.size_3",hhSize3.build()); + RModelDataBuilder.add("hh.size_23",hhSize3.build()); + RModelDataBuilder.add("hh.size_4",hhSize4.build()); + RModelDataBuilder.add("hh.size_5",hhSize5.build()); + RModelDataBuilder.add("hh.size_345",hhSize345.build()); + RModelDataBuilder.add("hh.size_45",hhSize45.build()); + + RModelDataBuilder.add("hh.pers_under6",hhPersons0to6.build()); + RModelDataBuilder.add("hh.pers_6to17",hhPersons6to17.build()); + RModelDataBuilder.add("hh.pers_18to29_w",hhPersons18to29_worker.build()); + RModelDataBuilder.add("hh.pers_18to29_s",hhPersons18to29_student.build()); + RModelDataBuilder.add("hh.pers_18to29_u",hhPersons18to29_unemployed.build()); + RModelDataBuilder.add("hh.pers_30to64_w",hhPersons30to64_worker.build()); + RModelDataBuilder.add("hh.pers_30to64_s",hhPersons30to64_student.build()); + RModelDataBuilder.add("hh.pers_18to64_s",hhPersons18to64_student.build()); + RModelDataBuilder.add("hh.pers_30to64_u",hhPersons30to64_unemployed.build()); + RModelDataBuilder.add("hh.pers_18to64_u",hhPersons18to64_unemployed.build()); + RModelDataBuilder.add("hh.pers_65up",hhPersons65up.build()); + + RModelDataBuilder.add("hh.pers_female",hhPersonsFemale.build()); + RModelDataBuilder.add("hh.pers_mobilityRestriction",hhPersonsWithMobilityRestriction.build()); + + RModelDataBuilder.add("hh.economicStatus",hhEconomicStatus.build()); + RModelDataBuilder.add("hh.economicStatus_2",hhEconomicStatus2.build()); + RModelDataBuilder.add("hh.economicStatus_3",hhEconomicStatus3.build()); + RModelDataBuilder.add("hh.economicStatus_23",hhEconomicStatus23.build()); + RModelDataBuilder.add("hh.economicStatus_4",hhEconomicStatus4.build()); + RModelDataBuilder.add("hh.economicStatus_5",hhEconomicStatus5.build()); + + RModelDataBuilder.add("hh.autos",hhAutos.build()); + RModelDataBuilder.add("hh.propAutos",hhPropAutos.build()); + + RModelDataBuilder.add("hh.BBSR",hhRegionType.build()); + RModelDataBuilder.add("hh.BBSR_2",hhRegionType2.build()); + RModelDataBuilder.add("hh.BBSR_3",hhRegionType3.build()); + RModelDataBuilder.add("hh.BBSR_4",hhRegionType4.build()); + + this.RDataFrame = RModelDataBuilder.build(); + + } + + + + + + + + diff --git a/src/main/java/de/tum/bgu/msm/modules/tripGeneration/HouseholdType.java b/src/main/java/de/tum/bgu/msm/modules/tripGeneration/HouseholdType.java deleted file mode 100644 index 6ca3450d..00000000 --- a/src/main/java/de/tum/bgu/msm/modules/tripGeneration/HouseholdType.java +++ /dev/null @@ -1,78 +0,0 @@ -package de.tum.bgu.msm.modules.tripGeneration; - -/** - * Created by Nico on 20.07.2017. - */ -public class HouseholdType { - - private final int sizeL; - private final int sizeH; - private final int workersL; - private final int workersH; - private final int economicStatusL; - private final int economicStatusH; - private final int autosL; - private final int autosH; - private final int regionL; - private final int regionH; - private final int id; - - private int numberOfRecords = 0; - - public HouseholdType(int id, int sizeL, int sizeH, int workersL, int workersH, int economicStatusL, int economicStatusH, int autosL, int autosH, int regionL, int regionH) { - this.id = id; - this.sizeL = sizeL; - this.sizeH = sizeH; - this.workersL = workersL; - this.workersH = workersH; - this.economicStatusL = economicStatusL; - this.economicStatusH = economicStatusH; - this.autosL = autosL; - this.autosH = autosH; - this.regionL = regionL; - this.regionH = regionH; - } - - public int getNumberOfRecords() { - return numberOfRecords; - } - - public int getId() { - return this.id; - } - - public boolean applies(int size, int workers, int economicStatus, int autos, int region) { - if (appliesInSize(size) && appliesInWorkers(workers) && appliesInEconomicStatus(economicStatus) && appliesInAutos(autos) && appliesInRegion(region)) { - numberOfRecords++; - return true; - } else { - return false; - } - } - - private boolean appliesInRegion(int region) { - return region >= regionL && region <= regionH; - } - - private boolean appliesInAutos(int autos) { - return autos >= autosL - && autos <= autosH; - } - - private boolean appliesInEconomicStatus(int economicStatus) { - return economicStatus >= economicStatusL && economicStatus <= economicStatusH; - } - - private boolean appliesInWorkers(int workers) { - return workers >= workersL && workers <= workersH; - } - - private boolean appliesInSize(int size) { - return size >= sizeL && size <= sizeH; - } - - public boolean hasTheseAttributes(int hhSizeL, int hhSizeH, int wrkL, int wrkH, int ecoStatL, int ecoStatH, int autoL, int autoH, int regL, int regH) { - return(hhSizeL==sizeL && hhSizeH==sizeH && wrkL==workersL && wrkH == workersH && ecoStatL == economicStatusL && - ecoStatH == economicStatusH && autoL==autosL && autoH==autosH && regL==regionL && regH==regionH); - } -} diff --git a/src/main/java/de/tum/bgu/msm/modules/tripGeneration/HouseholdTypeManager.java b/src/main/java/de/tum/bgu/msm/modules/tripGeneration/HouseholdTypeManager.java deleted file mode 100644 index d381fddb..00000000 --- a/src/main/java/de/tum/bgu/msm/modules/tripGeneration/HouseholdTypeManager.java +++ /dev/null @@ -1,164 +0,0 @@ -package de.tum.bgu.msm.modules.tripGeneration; - -import de.tum.bgu.msm.data.DataSet; -import de.tum.bgu.msm.data.MitoHousehold; -import de.tum.bgu.msm.data.Purpose; -import de.tum.bgu.msm.io.input.readers.GenericCsvReader; -import de.tum.bgu.msm.io.input.readers.GenericCsvReader.GenericCsvTable; -import de.tum.bgu.msm.resources.Resources; -import org.apache.log4j.Logger; - -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author Nico - * @date 20.07.2017 - */ -public class HouseholdTypeManager { - - private static final Logger logger = Logger.getLogger(HouseholdTypeManager.class); - - private final Purpose purpose; - private final Map tripFrequency; - - final List householdTypes = new ArrayList<>(); - - public HouseholdTypeManager(Purpose purpose) { - this.purpose = purpose; - createHouseHoldTypeDefinitions(); - tripFrequency = readTripFrequencies(); - } - - public Integer[] getTripFrequenciesForHouseholdType(HouseholdType ht) { - return tripFrequency.get(ht); - } - - private Map readTripFrequencies() { - Path filePath = Resources.instance.getTripFrequenciesFilePath(purpose); - GenericCsvReader csvReader = new GenericCsvReader(filePath); - csvReader.read(); - GenericCsvTable dataTable = csvReader.getTable(); - Map tripFrequency = new HashMap<>(); - - for (HouseholdType ht: householdTypes) { - List tripFrequencyThisHousehold = new ArrayList<>(); - boolean foundThisHhType = false; - for (int row = 0; row < dataTable.getRowCount(); row++) { - String purpose = dataTable.getString(row, dataTable.getColumnIndexOf("typePurpose")); - if (!purpose.equals(this.purpose.toString())) { - logger.error("File " + filePath + " contains trip purpose " + purpose + - ", which is different from expected purpose " + this.purpose); - throw new RuntimeException("File " + filePath + " contains trip purpose " + purpose + - ", which is different from expected purpose " + this.purpose); - } - - if (ht.hasTheseAttributes(dataTable.getInt(row, "hhSize_L"), - dataTable.getInt(row, "hhSize_H"), - dataTable.getInt(row, "workers_L"), - dataTable.getInt(row, "workers_H"), - dataTable.getInt(row, "econStatus_L"), - dataTable.getInt(row, "econStatus_H"), - dataTable.getInt(row, "autos_L"), - dataTable.getInt(row, "autos_H"), - dataTable.getInt(row, "region_L"), - dataTable.getInt(row, "region_H"))) { - foundThisHhType = true; - for (int trips = 0; trips < 100; trips++) { - String columnName = "trips_" + trips; - if (dataTable.containsColumn(columnName)) { - tripFrequencyThisHousehold.add(dataTable.getInt(row, columnName)); - } - } - tripFrequency.put(ht, tripFrequencyThisHousehold.toArray(new Integer[]{})); - } - } - if (!foundThisHhType) { - logger.error("Could not find household type " + ht.getId() + " in file " + filePath); - } - } - return tripFrequency; - } - - private void createHouseHoldTypeDefinitions() { - // todo: should this not read the token from the class Properties.java? - String[] householdDefinitionToken = Resources.instance.getArray("hh.type." + purpose); - String sizeToken = householdDefinitionToken[1]; - String[] sizePortions = sizeToken.split("\\."); - String workerToken = householdDefinitionToken[2]; - String[] workerPortions = workerToken.split("\\."); - String incomeToken = householdDefinitionToken[3]; - String[] economicStatusPortions = incomeToken.split("\\."); - String autoToken = householdDefinitionToken[4]; - String[] autoPortions = autoToken.split("\\."); - String regionToken = householdDefinitionToken[5]; - String[] regionPortions = regionToken.split("\\."); - - createHouseholdTypes(sizePortions, workerPortions, - economicStatusPortions, autoPortions, regionPortions); - } - - private void createHouseholdTypes(String[] sizePortions, String[] workerPortions, - String[] economicStatusPortions, String[] autoPortions, String[] regionPortions) { - int id = 0; - for (String sizeToken : sizePortions) { - String[] sizeParts = sizeToken.split("-"); - for (String workerToken : workerPortions) { - String[] workerParts = workerToken.split("-"); - for (String economicStatusToken : economicStatusPortions) { - String[] economicStatusParts = economicStatusToken.split("-"); - for (String autoToken : autoPortions) { - String[] autoParts = autoToken.split("-"); - for (String regionToken : regionPortions) { - String[] regionParts = regionToken.split("-"); - final int sizeL = Integer.parseInt(sizeParts[0]); - final int sizeH = Integer.parseInt(sizeParts[1]); - final int workersL = Integer.parseInt(workerParts[0]); - final int workersH = Integer.parseInt(workerParts[1]); - final int economicStatusL = Integer.parseInt(economicStatusParts[0]); - final int economicStatusH = Integer.parseInt(economicStatusParts[1]); - final int autosL = Integer.parseInt(autoParts[0]); - final int autosH = Integer.parseInt(autoParts[1]); - final int regionL = Integer.parseInt(regionParts[0]); - final int regionH = Integer.parseInt(regionParts[1]); - - householdTypes.add(new HouseholdType(id, sizeL, sizeH, workersL, workersH, - economicStatusL, economicStatusH, autosL, autosH, regionL, - regionH)); - id++; - } - } - } - } - } - } - - HouseholdType determineHouseholdType(MitoHousehold hh) { - int areaType = -1; - if (hh.getHomeZone() != null) { - areaType = hh.getHomeZone().getAreaTypeSG().code() / 10; - } else { - logger.info("Home MitoZone for Household " + hh.getId() + " is null!"); - } - return determineHouseholdType(hh.getHhSize(), DataSet.getNumberOfWorkersForHousehold(hh), - hh.getEconomicStatus(), hh.getAutos(), areaType); - } - - private HouseholdType determineHouseholdType(int hhSze, int hhWrk, int hhEconStatus, int hhVeh, int hhReg) { - - hhSze = Math.min(hhSze, 8); - hhWrk = Math.min(hhWrk, 4); - hhVeh = Math.min(hhVeh, 3); - - for (HouseholdType type : householdTypes) { - if (type.applies(hhSze, hhWrk, hhEconStatus, hhVeh, hhReg)) { - return type; - } - } - logger.error("Could not define household type: " + hhSze + " " + hhWrk + " " + hhEconStatus + " " + hhVeh + " " + hhReg); - return null; - } -} diff --git a/src/main/java/de/tum/bgu/msm/modules/tripGeneration/RawTripGenerator.java b/src/main/java/de/tum/bgu/msm/modules/tripGeneration/RawTripGenerator.java index e3ca9123..4e01725e 100644 --- a/src/main/java/de/tum/bgu/msm/modules/tripGeneration/RawTripGenerator.java +++ b/src/main/java/de/tum/bgu/msm/modules/tripGeneration/RawTripGenerator.java @@ -43,6 +43,8 @@ public void run (double scaleFactorForGeneration) { } private void generateByPurposeMultiThreaded(double scaleFactorForGeneration) { + logger.info("Building R data frame"); + dataSet.buildRdataFrame(); final ConcurrentExecutor>>> executor = ConcurrentExecutor.fixedPoolService(Purpose.values().length); List>>>> tasks = new ArrayList<>(); diff --git a/src/main/java/de/tum/bgu/msm/modules/tripGeneration/TripsByPurposeGenerator.java b/src/main/java/de/tum/bgu/msm/modules/tripGeneration/TripsByPurposeGenerator.java index c768e37f..1daaa061 100644 --- a/src/main/java/de/tum/bgu/msm/modules/tripGeneration/TripsByPurposeGenerator.java +++ b/src/main/java/de/tum/bgu/msm/modules/tripGeneration/TripsByPurposeGenerator.java @@ -7,8 +7,15 @@ import de.tum.bgu.msm.util.concurrent.RandomizableConcurrentFunction; import org.apache.log4j.Logger; import org.matsim.core.utils.collections.Tuple; +import org.renjin.primitives.matrix.Matrix; +import org.renjin.script.RenjinScriptEngineFactory; +import org.renjin.sexp.Vector; -import java.util.*; +import javax.script.ScriptEngine; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static de.tum.bgu.msm.modules.tripGeneration.RawTripGenerator.DROPPED_TRIPS_AT_BORDER_COUNTER; import static de.tum.bgu.msm.modules.tripGeneration.RawTripGenerator.TRIP_ID_COUNTER; @@ -16,6 +23,8 @@ class TripsByPurposeGenerator extends RandomizableConcurrentFunction>>> { private static final Logger logger = Logger.getLogger(TripsByPurposeGenerator.class); + private static final ThreadLocal ENGINE = new ThreadLocal<>(); + private final boolean dropAtBorder = Resources.instance.getBoolean(Properties.REMOVE_TRIPS_AT_BORDER); private Map> tripsByHH = new HashMap<>(); @@ -23,7 +32,6 @@ class TripsByPurposeGenerator extends RandomizableConcurrentFunction>> call() { + public Tuple>> call() throws Exception { logger.info(" Generating trips with purpose " + purpose + " (multi-threaded)"); logger.info("Created trip frequency distributions for " + purpose); logger.info("Started assignment of trips for hh, purpose: " + purpose); - final Iterator iterator = dataSet.getHouseholds().values().iterator(); - for (; iterator.hasNext(); ) { - MitoHousehold next = iterator.next(); - generateTripsForHousehold(next, scaleFactorForGeneration); - } + generateTripsInR(purpose); return new Tuple<>(purpose, tripsByHH); } + private void generateTripsInR(Purpose purpose) throws Exception { + + int numberOfHouseholds = dataSet.getHouseholds().size(); + Vector probabilityVector = null; + Matrix probabilityMatrix; + + ScriptEngine engine = ENGINE.get(); + + if(engine == null) { + // create a new engine for this thread + RenjinScriptEngineFactory factory = new RenjinScriptEngineFactory(); + engine = factory.getScriptEngine(); + + String modelFileName = "tripGenModel_" + purpose + ".rds"; + String modelFilePath = this.getClass().getResource(modelFileName).getPath(); + + logger.info("Sharing " + purpose + " MITO data with R"); + engine.put("modelFilePath",modelFilePath); + engine.put("model_data", dataSet.getRdataFrame()); - private void generateTripsForHousehold(MitoHousehold hh, double scaleFactorForGeneration) { - HouseholdType hhType = householdTypeManager.determineHouseholdType(hh); - if (hhType == null) { - logger.error("Could not create trips for Household " + hh.getId() + " for Purpose " + purpose + ": No Household Type applicable"); - return; + logger.info("Reading " + modelFileName + " into R"); + engine.eval("model <- readRDS(modelFilePath)"); + + logger.info("Building " + purpose + " probability matrix in R"); + engine.eval("probability_matrix <- predict(model, type = \"prob\", newdata = model_data)"); + +// Dump R output to a .rda file (this is an example for reading a largeR R script from resources folder) +// engine.put("purpose",purpose.toString()); +// engine.eval(new InputStreamReader(this.getClass().getResourceAsStream("tripGenModel.R"))); + + // Retrieve probability matrix & corresponding household IDs from R + probabilityVector = (Vector)engine.eval("probability_matrix"); + + ENGINE.set(engine); } - Integer[] tripFrequencies = householdTypeManager.getTripFrequenciesForHouseholdType(hhType); - if (tripFrequencies == null) { - logger.error("Could not find trip frequencies for this hhType/Purpose: " + hhType.getId() + "/" + purpose); - return; + + // Print details on probability matrices for each purpose + probabilityMatrix = new Matrix(probabilityVector); + try { + logger.info(purpose + ": Probability Matrix is a " + probabilityMatrix.getNumRows() + "x" + probabilityMatrix.getNumCols() + " matrix."); + } catch(IllegalArgumentException e) { + logger.info(purpose + ": Probability Matrix is not a matrix: " + e); + } + + // Get vector of household IDs that match probability rows of probability matrix + Vector hhIdVector = dataSet.getRdataFrame().getElementAsVector("hh.id"); + int probabilityMatrixCols = probabilityMatrix.getNumCols(); + + // Possible numbers of trips that can be chosen (column names of probability matrix) + int[] possibleSelections = new int[probabilityMatrixCols]; + for(int i = 0 ; i < probabilityMatrixCols ; i++) { + possibleSelections[i] = Integer.parseInt(probabilityMatrix.getColName(i)); } - if (MitoUtil.getSum(tripFrequencies) == 0) { - logger.info("No trips for this hhType/Purpose: " + hhType.getId() + "/" + purpose); - return; + + // Loop over householdIDs and probabilities and generate trips + logger.info("Assigning trips for purpose: " + purpose); + for(int i = 0 ; i < numberOfHouseholds ; i++ ) { + + int hhId = hhIdVector.getElementAsInt(i); + MitoHousehold hh = dataSet.getHouseholds().get(hhId); + + double[] tripFrequencies = new double[probabilityMatrixCols]; + for(int j = 0 ; j < probabilityMatrixCols ; j++) { + tripFrequencies[j] = probabilityMatrix.getElementAsDouble(i,j); + } + + int selection = MitoUtil.select(tripFrequencies); + int numberOfTrips = possibleSelections[selection]; + + if(hh.getHomeZone() != null) { + generateTripsForHousehold(hh, numberOfTrips); + } else { + logger.info("Couldn't generate trips for household " + hhId + " purpose " + purpose + ": no home zone"); + } + } + } + private void generateTripsForHousehold(MitoHousehold hh, int numberOfTrips) { List trips = new ArrayList<>(); - int numberOfTrips = selectNumberOfTrips(tripFrequencies); for (int i = 0; i < numberOfTrips; i++) { if (MitoUtil.getRandomObject().nextDouble() < scaleFactorForGeneration){ MitoTrip trip = createTrip(hh); @@ -74,19 +138,10 @@ private void generateTripsForHousehold(MitoHousehold hh, double scaleFactorForGe trips.add(trip); } } - } tripsByHH.put(hh, trips); } - private int selectNumberOfTrips(Integer[] tripFrequencies) { - double[] probabilities = new double[tripFrequencies.length]; - for (int i = 0; i < tripFrequencies.length; i++) { - probabilities[i] = (double) tripFrequencies[i]; - } - return MitoUtil.select(probabilities, random); - } - private MitoTrip createTrip(MitoHousehold hh) { boolean dropThisTrip = reduceTripGenAtStudyAreaBorder(hh.getHomeZone()); if (dropThisTrip) { diff --git a/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel.R b/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel.R new file mode 100644 index 00000000..cef50718 --- /dev/null +++ b/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel.R @@ -0,0 +1,6 @@ +# EXAMPLE CODE: Save everything to a .rda file (takes a long time!) +rdaFileName <- paste(purpose, "_modelData.rda", sep = "") +print(paste("R: Saving full image to ",getwd(),"/",rdaFileName)) +save.image(file = rdaFileName) +print(paste("R: purpose",purpose,"DONE")) + diff --git a/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_HBE.rds b/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_HBE.rds new file mode 100644 index 00000000..9fa77794 Binary files /dev/null and b/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_HBE.rds differ diff --git a/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_HBO.rds b/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_HBO.rds new file mode 100644 index 00000000..a9b196aa Binary files /dev/null and b/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_HBO.rds differ diff --git a/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_HBR.rds b/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_HBR.rds new file mode 100644 index 00000000..622a2379 Binary files /dev/null and b/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_HBR.rds differ diff --git a/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_HBS.rds b/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_HBS.rds new file mode 100644 index 00000000..a6a5758c Binary files /dev/null and b/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_HBS.rds differ diff --git a/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_HBW.rds b/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_HBW.rds new file mode 100644 index 00000000..a8923df2 Binary files /dev/null and b/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_HBW.rds differ diff --git a/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_NHBO.rds b/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_NHBO.rds new file mode 100644 index 00000000..31676f4f Binary files /dev/null and b/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_NHBO.rds differ diff --git a/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_NHBW.rds b/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_NHBW.rds new file mode 100644 index 00000000..fc720336 Binary files /dev/null and b/src/main/resources/de/tum/bgu/msm/modules/tripGeneration/tripGenModel_NHBW.rds differ diff --git a/src/test/java/de/tum/bgu/msm/modules/tripGeneration/HouseHoldTypeManagerTest.java b/src/test/java/de/tum/bgu/msm/modules/tripGeneration/HouseHoldTypeManagerTest.java deleted file mode 100644 index 48bdd172..00000000 --- a/src/test/java/de/tum/bgu/msm/modules/tripGeneration/HouseHoldTypeManagerTest.java +++ /dev/null @@ -1,47 +0,0 @@ -package de.tum.bgu.msm.modules.tripGeneration; - -import de.tum.bgu.msm.data.*; -import de.tum.bgu.msm.resources.Resources; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.util.List; - -/** - * Created by Nico on 26/07/2017. - */ -public class HouseHoldTypeManagerTest { - - private MitoZone zone; - - @Before - public void setupTest() { - Resources.initializeResources("./test/muc/test.properties"); - - zone = new MitoZone(1, AreaTypes.SGType.RURAL); - } - - @Test - public final void test() { - HouseholdTypeManager manager = new HouseholdTypeManager(Purpose.HBW); - List types = manager.householdTypes; - Assert.assertEquals(20, types.size()); - for(HouseholdType type: types) { - Assert.assertEquals(0, type.getNumberOfRecords()); - } - - MitoHousehold household1 = new MitoHousehold(1, 4, 1); - household1.setHomeZone(new MitoZone(1, AreaTypes.SGType.CORE_CITY)); - household1.addPerson(new MitoPerson(1, MitoOccupationStatus.WORKER, null, 30, MitoGender.MALE, true)); - MitoHousehold household2 = new MitoHousehold(2, 4, 1); - household2.setHomeZone(zone); - household2.addPerson(new MitoPerson(2, MitoOccupationStatus.WORKER, null, 30, MitoGender.MALE, true)); - Assert.assertNull(manager.determineHouseholdType(household1)); - - household2.setEconomicStatus(1); - HouseholdType determinedType = manager.determineHouseholdType(household2); - Assert.assertNotNull(determinedType); - Assert.assertEquals(1, determinedType.getNumberOfRecords()); - } -}