From dfd1ec9f8b8866ae2300da34c57d9c41724b3d41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torbj=C3=B8rn=20Barslett?= Date: Fri, 20 Jan 2017 05:53:43 +0100 Subject: [PATCH 01/12] Fixed NullPointerException if clustering takes place before graph is created (#2394) --- .../routing/graph/GraphIndex.java | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/opentripplanner/routing/graph/GraphIndex.java b/src/main/java/org/opentripplanner/routing/graph/GraphIndex.java index ad583ac216c..c16a04b355a 100644 --- a/src/main/java/org/opentripplanner/routing/graph/GraphIndex.java +++ b/src/main/java/org/opentripplanner/routing/graph/GraphIndex.java @@ -563,16 +563,20 @@ public Timetable currentUpdatedTimetableForTripPattern (TripPattern tripPattern) * stops -- no guessing is reasonable without that information. */ public void clusterStops() { - switch (graph.stopClusterMode) { - case "parentStation": - clusterByParentStation(); - break; - case "proximity": - clusterByProximity(); - break; - default: - clusterByProximity(); - } + if (graph.stopClusterMode != null) { + switch (graph.stopClusterMode) { + case "parentStation": + clusterByParentStation(); + break; + case "proximity": + clusterByProximity(); + break; + default: + clusterByProximity(); + } + } else { + clusterByProximity(); + } } private void clusterByProximity() { From 9eda9c80448cdf96db5857bad3ea8ec22ca844fc Mon Sep 17 00:00:00 2001 From: Stefan de Konink Date: Mon, 2 May 2016 03:51:44 +0200 Subject: [PATCH 02/12] First implementation of the DutchFareService. This service requires proprietary fare_attributes.txt and fare_rules.txt brought to you by Bliksem Labs B.V. --- .../routing/impl/DutchFareServiceFactory.java | 23 ++ .../routing/impl/DutchFareServiceImpl.java | 258 ++++++++++++++++++ 2 files changed, 281 insertions(+) create mode 100644 src/main/java/org/opentripplanner/routing/impl/DutchFareServiceFactory.java create mode 100644 src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java diff --git a/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceFactory.java b/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceFactory.java new file mode 100644 index 00000000000..f7f2d815fef --- /dev/null +++ b/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceFactory.java @@ -0,0 +1,23 @@ +/* This program is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License + as published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . */ + +package org.opentripplanner.routing.impl; + +import org.opentripplanner.routing.services.FareService; + +public class DutchFareServiceFactory extends DefaultFareServiceFactory { + @Override + public FareService makeFareService() { + return new DutchFareServiceImpl(regularFareRules.values()); + } +} diff --git a/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java b/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java new file mode 100644 index 00000000000..78db05d669d --- /dev/null +++ b/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java @@ -0,0 +1,258 @@ +/* This program is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License + as published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . */ + +package org.opentripplanner.routing.impl; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.geotools.xml.xsi.XSISimpleTypes; +import org.onebusaway.gtfs.model.AgencyAndId; +import org.opentripplanner.common.model.P2; +import org.opentripplanner.routing.core.FareRuleSet; +import org.opentripplanner.routing.core.Fare.FareType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DutchFareServiceImpl extends DefaultFareServiceImpl { + + public DutchFareServiceImpl(Collection regularFareRules) { + addFareRules(FareType.regular, regularFareRules); + } + + private static final long serialVersionUID = 1L; + @SuppressWarnings("unused") + private static final Logger LOG = LoggerFactory.getLogger(DutchFareServiceImpl.class); + + public static final int TRANSFER_DURATION = 60 * 35; /* tranfers within 35 min won't require a new base fare */ + + /* The Netherlands has an almost uniform system for electronic ticketing using a NFC-card, branded as OV-chipkaart. + * + * To travel through all modes in The Netherlands a uses has two products on their card: + * 1) Easy Trip, to travel with all operators except Dutch Railways + * 2) Reizen op Saldo, for Dutch Railways which should be explicitly + * loaded on a card and requires the user to select a first or second class. + * + * Check-in and check-out is done using validators. For our calculation on transerfer time it matters if this + * validator is inside the vehicle - we have to wait for the validator to arrive - or we can already check in + * on a validator present on the stop, and wait for the vehicle to arrive. + * + * Reizen op Saldo is limited to Dutch Railways, and always allows to validate inside the stations. + * Additionally in the following cases validators are also on the platform or stations. + * - Metro of Amsterdam operated by GVB + * - Metro of Rotterdam operated by RET + * - Lightrail of Utrecht operated by Qbuzz + * - All heavy rail services operated by Arriva, Breng, Connexxion, NS, Syntus and Veolia. + * + * Leaving the platform or stations implies that the traveler must check-out. Thus a transfer will play a role. + * + * All other modes by these operators have validators inside the vehicle. + * + * TODO: It is an optimisation to be able to check-in early. And most likely only be visible by a trip which is + * artificially created to test for this implementation. + * + * Long-Distance-Discount for trains + * Between train operators in The Netherlands long distance discount applies under the following condition: + * - a trip between two train operators takes places within 35 minutes + * + * First the route is maximised per operator. + * + * The price of the next operator consists of: + * globally traveled units = 100 previous operator(s) + * locally traveled units = 10 + * + * (DutchRailwaysPrice(0 + 100) - DutchRailwaysPrice(0)) + (ArrivaPrice(100 + 10) - ArrivaPrice(100)) + */ + + private class UnitsFareZone { + public int units; + public String fareZone; + + public UnitsFareZone(int units, String fareZone) { + this.units = units; + this.fareZone = fareZone; + } + } + + private UnitsFareZone getUnitsByZones(String startZone, String endZone, Collection fareRules) { + P2 od = new P2(startZone, endZone); + + LOG.warn("Search " + startZone + " and " + endZone); + + for (FareRuleSet ruleSet : fareRules) { + /* TODO: agency selectie moet hier ook nog, dordrecht geldermalsen */ + if (ruleSet.getOriginDestinations().contains(od)) { + String fareId = ruleSet.getFareAttribute().getId().getId(); + String[] parts = fareId.split(":"); + String fareZone = parts[0] + ':' + parts[1]; + + LOG.warn("Between " + startZone + " and " + endZone + ": " + (int) ruleSet.getFareAttribute().getPrice() + " (" + fareZone + ")"); + return new UnitsFareZone((int) ruleSet.getFareAttribute().getPrice(), fareZone); + } + } + + LOG.warn("Can't find units between " + startZone + " and " + endZone); + + /* TODO: Raise Exception */ + + return null; + } + + private float getCostByUnits(String fareZone, int units, int prevSumUnits, Collection fareRules) { + if (units == 0) { + return 0f; + } + + float cost = 0f; + + String fareId = fareZone + ":" + units; + for (FareRuleSet ruleSet : fareRules) { + if (ruleSet.getFareAttribute().getId().getId().equals(fareId)) { + cost = ruleSet.getFareAttribute().getPrice(); + break; + } + } + + if (cost == 0f) { + LOG.warn("Can't find price for " + fareZone + " with " + units + " units"); + + } else if (prevSumUnits > 0) { + + fareId = fareZone + ":" + prevSumUnits; + for (FareRuleSet ruleSet : fareRules) { + if (ruleSet.getFareAttribute().getId().getId().equals(fareId)) { + cost -= ruleSet.getFareAttribute().getPrice(); + break; + } + } + + LOG.warn("Can't find price for " + fareZone + " with " + prevSumUnits + " units"); + } + + return cost; + } + + private float getEasyTripFareByLineFromTo(AgencyAndId route, String firstStop, String lastStop, + boolean entranceFee, Collection fareRules) { + + float cost = 0f; + + for (FareRuleSet ruleSet : fareRules) { + Set agencies = new HashSet<>(); + agencies.add(route.getAgencyId()); + + Set routes = new HashSet<>(); + routes.add(route); + + if (ruleSet.matches(agencies, firstStop, lastStop, new HashSet(), routes)) { + cost = ruleSet.getFareAttribute().getPrice(); + break; + } + } + + if (entranceFee) cost += 0.89f; /* TODO: Configurable? */ + + return cost; + } + + @Override + protected float getLowestCost(FareType fareType, List rides, + Collection fareRules) { + + float cost = 0f; + + int units = 0; + int prevSumUnits = 0; + + boolean mustHaveCheckedOut = false; + String startTariefEenheden = null; + String endTariefEenheden = null; + String lastAgencyId = null; + String lastFareZone = null; + + long alightedEasyTrip = 0; + long alightedTariefEenheden = 0; + + for (Ride ride : rides) { + LOG.warn(String.format("%s %s %s %s %s %s", ride.startZone, ride.endZone, ride.firstStop, ride.lastStop, ride.route, ride.agency)); + + if (ride.agency.startsWith("IFF:")) { + LOG.warn("1. NS"); + /* In Reizen op Saldo we will try to fares as long as possible. */ + + /* If our previous agency isn't this agency, then we must have checked out */ + mustHaveCheckedOut |= !ride.agency.equals(lastAgencyId); + + /* When a user has checked out, we first calculate the units made until then. */ + if (mustHaveCheckedOut && lastAgencyId != null) { + LOG.warn("2. Must Have Checked Out"); + UnitsFareZone unitsFareZone = getUnitsByZones(startTariefEenheden, endTariefEenheden, fareRules); + if (unitsFareZone == null) return Float.POSITIVE_INFINITY; + lastFareZone = unitsFareZone.fareZone; + units += unitsFareZone.units; + startTariefEenheden = ride.startZone; + mustHaveCheckedOut = false; + } + + /* The entrance Fee applies if the transfer time ends before the new trip starts. */ + if ((alightedTariefEenheden + TRANSFER_DURATION) < ride.startTime) { + LOG.warn("3. Exceeded Transfer Time"); + cost += getCostByUnits(lastFareZone, units, prevSumUnits, fareRules); + startTariefEenheden = ride.startZone; + units = 0; + prevSumUnits = 0; + mustHaveCheckedOut = false; + + } else if (!ride.agency.equals(lastAgencyId)) { + LOG.warn("4. Swiched Rail Agency"); + + cost += getCostByUnits(lastFareZone, units, prevSumUnits, fareRules); + units = 0; + startTariefEenheden = ride.startZone; + } + + alightedTariefEenheden = ride.endTime; + endTariefEenheden = ride.endZone; + lastAgencyId = ride.agency; + + } else { + LOG.warn("5. Easy Trip"); + + /* We are now on Easy Trip, so we must have checked-out from Reizen op Saldo, if we were on it */ + mustHaveCheckedOut = (startTariefEenheden != null); + + /* The entranceFee applies if the transfer time ends before the new trip starts. */ + boolean entranceFee = ((alightedEasyTrip + TRANSFER_DURATION) < ride.startTime); + + /* EasyTrip will always calculate its price per leg */ + cost += getEasyTripFareByLineFromTo(ride.route, ride.firstStop.toString(), ride.lastStop.toString(), entranceFee, fareRules); + + alightedEasyTrip = ride.endTime; + } + } + + LOG.warn("6. Final"); + UnitsFareZone unitsFareZone = getUnitsByZones(startTariefEenheden, endTariefEenheden, fareRules); + if (unitsFareZone == null) return Float.POSITIVE_INFINITY; + + lastFareZone = unitsFareZone.fareZone; + units += unitsFareZone.units; + cost += getCostByUnits(lastFareZone, units, prevSumUnits, fareRules); + + return cost / 100f; + } +} From c33c44e8e20925ff029b9197b937551d70e72846 Mon Sep 17 00:00:00 2001 From: Stefan de Konink Date: Mon, 2 May 2016 03:53:40 +0200 Subject: [PATCH 03/12] Hotwire the DutchFareService so it can used in a build.config { fares: { type: "dutch", currency: "EUR" } } --- .../routing/impl/DefaultFareServiceFactory.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/opentripplanner/routing/impl/DefaultFareServiceFactory.java b/src/main/java/org/opentripplanner/routing/impl/DefaultFareServiceFactory.java index 975c4a2759d..f8a10759177 100644 --- a/src/main/java/org/opentripplanner/routing/impl/DefaultFareServiceFactory.java +++ b/src/main/java/org/opentripplanner/routing/impl/DefaultFareServiceFactory.java @@ -35,9 +35,9 @@ the License, or (props, at your option) any later version. /** * Implements the default GTFS fare rules as described in * http://groups.google.com/group/gtfs-changes/msg/4f81b826cb732f3b - * + * * @author novalis - * + * */ public class DefaultFareServiceFactory implements FareServiceFactory { @@ -111,9 +111,9 @@ public void configure(JsonNode config) { /** * Build a specific FareServiceFactory given the config node, or fallback to the default if none * specified. - * + * * Accept different formats. Examples: - * + * *
      * { fares : "seattle" }
      * --------------------------
@@ -172,6 +172,9 @@ public static FareServiceFactory fromConfig(JsonNode config) {
         case "bike-rental-time-based":
             retval = new TimeBasedBikeRentalFareServiceFactory();
             break;
+        case "dutch":
+            retval = new DutchFareServiceFactory();
+            break;
         case "san-francisco":
             retval = new SFBayFareServiceFactory();
             break;

From 6eff71b131ef89dce7d553cf86e4515c714ccd2c Mon Sep 17 00:00:00 2001
From: Stefan de Konink 
Date: Mon, 2 May 2016 03:56:05 +0200
Subject: [PATCH 04/12] Hardcode Euro's to the DutchFare branch.

OpenTripPlanner now fetches a random fare_attribute and assumes the entire feed is in this currency. It also ignores the build.config. Since the DutchFare table contains Distance Units in XXX currency it might (and thus: will) happen that XXX as currency ends up in the client.
---
 .../routing/impl/DefaultFareServiceImpl.java              | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/opentripplanner/routing/impl/DefaultFareServiceImpl.java b/src/main/java/org/opentripplanner/routing/impl/DefaultFareServiceImpl.java
index 0436fb66045..d53a3888902 100644
--- a/src/main/java/org/opentripplanner/routing/impl/DefaultFareServiceImpl.java
+++ b/src/main/java/org/opentripplanner/routing/impl/DefaultFareServiceImpl.java
@@ -217,13 +217,13 @@ public Fare getCost(GraphPath path) {
             FareType fareType = kv.getKey();
             Collection fareRules = kv.getValue();
 
-            // pick up a random currency from fareAttributes,
-            // we assume that all tickets use the same currency
+            // hardcode the fareAttribute currency, because the configuration
+            // parameter is ignored, and the assumption that all tickets are in the same currency
+            // doesn't hold for a feed with distance units
             Currency currency = null;
             WrappedCurrency wrappedCurrency = null;
             if (fareRules.size() > 0) {
-                currency = Currency.getInstance(fareRules.iterator().next().getFareAttribute()
-                        .getCurrencyType());
+                currency = Currency.getInstance("EUR");
                 wrappedCurrency = new WrappedCurrency(currency);
             }
             hasFare = populateFare(fare, currency, fareType, rides, fareRules);

From 1078c961210e77e444fc8ebe77257e14848c0d57 Mon Sep 17 00:00:00 2001
From: Stefan de Konink 
Date: Mon, 2 May 2016 05:36:09 +0200
Subject: [PATCH 05/12] Fix LAK.

---
 .../org/opentripplanner/routing/impl/DutchFareServiceImpl.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java b/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java
index 78db05d669d..a3237cceaf1 100644
--- a/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java
+++ b/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java
@@ -119,7 +119,7 @@ private float getCostByUnits(String fareZone, int units, int prevSumUnits, Colle
 
         float cost = 0f;
 
-        String fareId = fareZone + ":" + units;
+        String fareId = fareZone + ":" + (units + prevSumUnits);
         for (FareRuleSet ruleSet : fareRules) {
             if (ruleSet.getFareAttribute().getId().getId().equals(fareId)) {
                 cost = ruleSet.getFareAttribute().getPrice();
@@ -221,6 +221,7 @@ protected float getLowestCost(FareType fareType, List rides,
                     LOG.warn("4. Swiched Rail Agency");
 
                     cost += getCostByUnits(lastFareZone, units, prevSumUnits, fareRules);
+                    prevSumUnits += units;
                     units = 0;
                     startTariefEenheden = ride.startZone;
                 }

From 3f25ce1133c320f0e2a0373632ed0a62202ec61b Mon Sep 17 00:00:00 2001
From: Stefan de Konink 
Date: Tue, 3 May 2016 21:04:40 +0200
Subject: [PATCH 06/12] Fix entrance fee, now in cents. Add warning.

---
 .../opentripplanner/routing/impl/DutchFareServiceImpl.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java b/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java
index a3237cceaf1..5a0c4583e4b 100644
--- a/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java
+++ b/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java
@@ -162,9 +162,11 @@ private float getEasyTripFareByLineFromTo(AgencyAndId route, String firstStop, S
                 cost = ruleSet.getFareAttribute().getPrice();
                 break;
             }
+
+            LOG.warn("Can't find price for " + firstStop + " to " + lastStop + " operated on " + route.getId());
         }
 
-        if (entranceFee) cost += 0.89f; /* TODO: Configurable? */
+        if (entranceFee) cost += 89f; /* TODO: Configurable? */
 
         return cost;
     }

From 75804dc08f2d1281d1d17ee98bdd89ccb0c261e0 Mon Sep 17 00:00:00 2001
From: Stefan de Konink 
Date: Tue, 3 May 2016 22:03:07 +0200
Subject: [PATCH 07/12] Include agency in the unit search. Fixes Dordrecht -
 Geldermalsen (if Arriva would provide the data)

---
 .../routing/impl/DutchFareServiceImpl.java       | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java b/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java
index 5a0c4583e4b..e2efff6d602 100644
--- a/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java
+++ b/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java
@@ -88,17 +88,19 @@ public UnitsFareZone(int units, String fareZone) {
         }
     }
 
-    private UnitsFareZone getUnitsByZones(String startZone, String endZone, Collection fareRules) {
+    private UnitsFareZone getUnitsByZones(String agencyId, String startZone, String endZone, Collection fareRules) {
         P2 od = new P2(startZone, endZone);
 
         LOG.warn("Search " + startZone + " and " + endZone);
 
+        String fareIdStartsWith = agencyId + "::";
+
         for (FareRuleSet ruleSet : fareRules) {
-            /* TODO: agency selectie moet hier ook nog, dordrecht geldermalsen */
-            if (ruleSet.getOriginDestinations().contains(od)) {
+            if (ruleSet.getFareAttribute().getId().getId().startsWith(fareIdStartsWith) &&
+                ruleSet.getOriginDestinations().contains(od)) {
                 String fareId = ruleSet.getFareAttribute().getId().getId();
-                String[] parts = fareId.split(":");
-                String fareZone = parts[0] + ':' + parts[1];
+                String[] parts = fareId.split("::");
+                String fareZone = parts[1];
 
                 LOG.warn("Between " + startZone + " and " + endZone + ": " + (int) ruleSet.getFareAttribute().getPrice() + " (" + fareZone + ")");
                 return new UnitsFareZone((int) ruleSet.getFareAttribute().getPrice(), fareZone);
@@ -202,7 +204,7 @@ protected float getLowestCost(FareType fareType, List rides,
                 /* When a user has checked out, we first calculate the units made until then. */
                 if (mustHaveCheckedOut && lastAgencyId != null) {
                     LOG.warn("2. Must Have Checked Out");
-                    UnitsFareZone unitsFareZone = getUnitsByZones(startTariefEenheden, endTariefEenheden, fareRules);
+                    UnitsFareZone unitsFareZone = getUnitsByZones(lastAgencyId, startTariefEenheden, endTariefEenheden, fareRules);
                     if (unitsFareZone == null) return Float.POSITIVE_INFINITY;
                     lastFareZone = unitsFareZone.fareZone;
                     units += unitsFareZone.units;
@@ -249,7 +251,7 @@ protected float getLowestCost(FareType fareType, List rides,
         }
 
         LOG.warn("6. Final");
-        UnitsFareZone unitsFareZone = getUnitsByZones(startTariefEenheden, endTariefEenheden, fareRules);
+        UnitsFareZone unitsFareZone = getUnitsByZones(lastAgencyId, startTariefEenheden, endTariefEenheden, fareRules);
         if (unitsFareZone == null) return Float.POSITIVE_INFINITY;
 
         lastFareZone = unitsFareZone.fareZone;

From bde11bebadcf67f3478592110826b2566b831ca4 Mon Sep 17 00:00:00 2001
From: Stefan de Konink 
Date: Wed, 4 May 2016 03:56:04 +0200
Subject: [PATCH 08/12] Updated for a working initial implementation.

---
 .../routing/impl/DutchFareServiceImpl.java    | 47 ++++++++++++-------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java b/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java
index e2efff6d602..138bc71d224 100644
--- a/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java
+++ b/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java
@@ -143,29 +143,31 @@ private float getCostByUnits(String fareZone, int units, int prevSumUnits, Colle
             }
 
             LOG.warn("Can't find price for " + fareZone + " with " + prevSumUnits + " units");
+
+            return Float.POSITIVE_INFINITY;
         }
 
         return cost;
     }
 
-    private float getEasyTripFareByLineFromTo(AgencyAndId route, String firstStop, String lastStop,
+    private float getEasyTripFareByLineFromTo(String route, String firstStop, String lastStop,
                                               boolean entranceFee, Collection fareRules) {
 
-        float cost = 0f;
-
-        for (FareRuleSet ruleSet : fareRules) {
-            Set agencies = new HashSet<>();
-            agencies.add(route.getAgencyId());
+        float cost = Float.POSITIVE_INFINITY;
 
-            Set routes = new HashSet<>();
-            routes.add(route);
+        String fareId = route + ":" + firstStop + "-" + lastStop;
 
-            if (ruleSet.matches(agencies, firstStop, lastStop, new HashSet(), routes)) {
+        for (FareRuleSet ruleSet : fareRules) {
+            if (ruleSet.getFareAttribute().getId().getId().equals(fareId)) {
                 cost = ruleSet.getFareAttribute().getPrice();
                 break;
             }
+        }
+
+        if (cost == Float.POSITIVE_INFINITY) {
+            LOG.warn("Can't find price for " + firstStop + " to " + lastStop + " operated on " + route);
 
-            LOG.warn("Can't find price for " + firstStop + " to " + lastStop + " operated on " + route.getId());
+            return cost;
         }
 
         if (entranceFee) cost += 89f; /* TODO: Configurable? */
@@ -195,7 +197,7 @@ protected float getLowestCost(FareType fareType, List rides,
             LOG.warn(String.format("%s %s %s %s %s %s", ride.startZone, ride.endZone, ride.firstStop, ride.lastStop, ride.route, ride.agency));
 
             if (ride.agency.startsWith("IFF:")) {
-                LOG.warn("1. NS");
+                LOG.warn("1. Trains");
 		        /* In Reizen op Saldo we will try to fares as long as possible. */
 
                 /* If our previous agency isn't this agency, then we must have checked out */
@@ -203,7 +205,7 @@ protected float getLowestCost(FareType fareType, List rides,
 
                 /* When a user has checked out, we first calculate the units made until then. */
                 if (mustHaveCheckedOut && lastAgencyId != null) {
-                    LOG.warn("2. Must Have Checked Out");
+                    LOG.warn("2. Must have checked out from a station");
                     UnitsFareZone unitsFareZone = getUnitsByZones(lastAgencyId, startTariefEenheden, endTariefEenheden, fareRules);
                     if (unitsFareZone == null) return Float.POSITIVE_INFINITY;
                     lastFareZone = unitsFareZone.fareZone;
@@ -216,6 +218,8 @@ protected float getLowestCost(FareType fareType, List rides,
                 if ((alightedTariefEenheden + TRANSFER_DURATION) < ride.startTime) {
                     LOG.warn("3. Exceeded Transfer Time");
                     cost += getCostByUnits(lastFareZone, units, prevSumUnits, fareRules);
+                    if (cost == Float.POSITIVE_INFINITY) return cost;
+
                     startTariefEenheden = ride.startZone;
                     units = 0;
                     prevSumUnits = 0;
@@ -225,6 +229,8 @@ protected float getLowestCost(FareType fareType, List rides,
                     LOG.warn("4. Swiched Rail Agency");
 
                     cost += getCostByUnits(lastFareZone, units, prevSumUnits, fareRules);
+                    if (cost == Float.POSITIVE_INFINITY) return cost;
+
                     prevSumUnits += units;
                     units = 0;
                     startTariefEenheden = ride.startZone;
@@ -244,19 +250,24 @@ protected float getLowestCost(FareType fareType, List rides,
                 boolean entranceFee = ((alightedEasyTrip + TRANSFER_DURATION) < ride.startTime);
 
                 /* EasyTrip will always calculate its price per leg */
-                cost += getEasyTripFareByLineFromTo(ride.route, ride.firstStop.toString(), ride.lastStop.toString(), entranceFee, fareRules);
+                cost += getEasyTripFareByLineFromTo(ride.route.getId(), ride.startZone, ride.endZone, entranceFee, fareRules);
+                if (cost == Float.POSITIVE_INFINITY) return cost;
 
                 alightedEasyTrip = ride.endTime;
             }
         }
 
         LOG.warn("6. Final");
-        UnitsFareZone unitsFareZone = getUnitsByZones(lastAgencyId, startTariefEenheden, endTariefEenheden, fareRules);
-        if (unitsFareZone == null) return Float.POSITIVE_INFINITY;
+        if (lastAgencyId != null) {
+            UnitsFareZone unitsFareZone = getUnitsByZones(lastAgencyId, startTariefEenheden, endTariefEenheden, fareRules);
+            if (unitsFareZone == null) return Float.POSITIVE_INFINITY;
+
+            lastFareZone = unitsFareZone.fareZone;
+            units += unitsFareZone.units;
+            cost += getCostByUnits(lastFareZone, units, prevSumUnits, fareRules);
+        }
 
-        lastFareZone = unitsFareZone.fareZone;
-        units += unitsFareZone.units;
-        cost += getCostByUnits(lastFareZone, units, prevSumUnits, fareRules);
+        if (cost == Float.POSITIVE_INFINITY) return cost;
 
         return cost / 100f;
     }

From 6930c4ef13f15b8491494aef994dbab124305841 Mon Sep 17 00:00:00 2001
From: Stefan de Konink 
Date: Sat, 10 Sep 2016 16:50:53 +0200
Subject: [PATCH 09/12] Rail units will not exceed 250.

http://wiki.ovinnederland.nl/wiki/Tariefeenheid#Tarieven_NS
---
 .../opentripplanner/routing/impl/DutchFareServiceImpl.java   | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java b/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java
index 138bc71d224..9659b128efe 100644
--- a/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java
+++ b/src/main/java/org/opentripplanner/routing/impl/DutchFareServiceImpl.java
@@ -119,6 +119,11 @@ private float getCostByUnits(String fareZone, int units, int prevSumUnits, Colle
             return 0f;
         }
 
+        /* Train-units cannot exceed 250 units; http://wiki.ovinnederland.nl/wiki/Tariefeenheid#Tarieven_NS */
+        if (units > 250) {
+            units = 250;
+        }
+
         float cost = 0f;
 
         String fareId = fareZone + ":" + (units + prevSumUnits);

From a602b5cbbd47fbad7b2e8875b343365d39d4871e Mon Sep 17 00:00:00 2001
From: Milena Mandic 
Date: Wed, 25 Jan 2017 16:38:17 +0100
Subject: [PATCH 10/12] Web service for status on real time updates v.1

---
 src/client/images/agency_logo.png             | Bin
 .../api/resource/UpdaterStatusResource.java   | 163 +++++++-
 .../opentripplanner/routing/graph/Graph.java  |  14 +-
 .../opentripplanner/standalone/OTPMain.java   |   2 +-
 .../updater/GraphUpdaterManager.java          | 253 ++++++++++-
 .../GtfsRealtimeHttpTripUpdateSource.java     |   8 +-
 .../stoptime/TimetableSnapshotSource.java     | 394 ++++++++++++++++--
 .../TripUpdateGraphWriterRunnable.java        |   5 +
 .../WebsocketGtfsRealtimeUpdater.java         |   4 +-
 src/scripts/otp                               |   0
 src/scripts/otp-batch-analyst                 |   0
 src/test/resources/generateGtfs.py            |   0
 src/test/resources/portland/Streets_pdx.shx   | Bin
 13 files changed, 774 insertions(+), 69 deletions(-)
 mode change 100755 => 100644 src/client/images/agency_logo.png
 mode change 100755 => 100644 src/scripts/otp
 mode change 100755 => 100644 src/scripts/otp-batch-analyst
 mode change 100755 => 100644 src/test/resources/generateGtfs.py
 mode change 100755 => 100644 src/test/resources/portland/Streets_pdx.shx

diff --git a/src/client/images/agency_logo.png b/src/client/images/agency_logo.png
old mode 100755
new mode 100644
diff --git a/src/main/java/org/opentripplanner/api/resource/UpdaterStatusResource.java b/src/main/java/org/opentripplanner/api/resource/UpdaterStatusResource.java
index 0f798ca3699..b22e0d84377 100644
--- a/src/main/java/org/opentripplanner/api/resource/UpdaterStatusResource.java
+++ b/src/main/java/org/opentripplanner/api/resource/UpdaterStatusResource.java
@@ -1,5 +1,6 @@
 package org.opentripplanner.api.resource;
 
+import com.google.transit.realtime.GtfsRealtime;
 import org.opentripplanner.standalone.OTPServer;
 import org.opentripplanner.standalone.Router;
 import org.opentripplanner.updater.GraphUpdater;
@@ -19,6 +20,7 @@
 @Produces(MediaType.APPLICATION_JSON)
 public class UpdaterStatusResource {
 
+
     private static final Logger LOG = LoggerFactory.getLogger(UpdaterStatusResource.class);
 
     /** Choose short or long form of results. */
@@ -30,29 +32,172 @@ public UpdaterStatusResource (@Context OTPServer otpServer, @PathParam("routerId
         router = otpServer.getRouter(routerId);
     }
 
+    /**
+     *
+     * @return most of the important data calculated by TimetableSnapshotSource
+     */
+    @GET
+    public Response getDescription() {
+        GraphUpdaterManager updaterManager = router.graph.updaterManager;
+        if (updaterManager == null) {
+            return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build();
+        }
+        return Response.status(Response.Status.OK).entity(updaterManager.getDescription()).build();
+    }
+
     /** Return a list of all agencies in the graph. */
     @GET
-    public Response getUpdaters () {
+    @Path("/agency/{feedId}")
+    public Response getAgencies (@PathParam("feedId") String feedId) {
         GraphUpdaterManager updaterManager = router.graph.updaterManager;
         if (updaterManager == null) {
-            return Response.status(Response.Status.NOT_FOUND).entity("No updaters running.").build();
+            return Response.status(Response.Status.NOT_FOUND).entity("There is no updaters running.").build();
         }
-        return Response.status(Response.Status.OK).entity(updaterManager.getUpdaterDescriptions()).build();
+        return Response.status(Response.Status.OK).entity(updaterManager.getAgency(feedId)).build();
     }
 
+    //TODO does not look good
     /** Return status for a specific updater. */
     @GET
     @Path("/{updaterId}")
-    public Response getUpdaters (@PathParam("updaterId") int updaterId) {
+    public Response getTypeOfUpdater (@PathParam("updaterId") int updaterId) {
+        GraphUpdaterManager updaterManager = router.graph.updaterManager;
+        if (updaterManager == null) {
+            return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build();
+        }
+        return Response.status(Response.Status.OK).entity(updaterManager.getUpdater(updaterId)).build();
+    }
+
+
+    //TODO
+    @GET
+    @Path("/stream")
+    public Response getStreamAddresses() {
         GraphUpdaterManager updaterManager = router.graph.updaterManager;
         if (updaterManager == null) {
-            return Response.status(Response.Status.NOT_FOUND).entity("No updaters running.").build();
+            return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build();
         }
-        GraphUpdater updater = updaterManager.getUpdater(updaterId);
-        if (updater == null) {
-            return Response.status(Response.Status.NOT_FOUND).entity("No updater with that ID.").build();
+        return Response.status(Response.Status.OK).entity(updaterManager.getStreamAddresses()).build();
+    }
+
+
+    @GET
+    @Path("/types")
+    public Response getTypes() {
+        GraphUpdaterManager updaterManager = router.graph.updaterManager;
+        if (updaterManager == null) {
+            return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build();
         }
-        return Response.status(Response.Status.OK).entity(updater.getClass()).build();
+        return Response.status(Response.Status.OK).entity(updaterManager.getAllTypes()).build();
     }
 
+    //TODO all updates
+    @GET
+    @Path("/updates")
+    public Response getUpdates () {
+        GraphUpdaterManager updaterManager = router.graph.updaterManager;
+        if (updaterManager == null) {
+            return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build();
+        }
+        return Response.status(Response.Status.OK).entity(updaterManager.getUpdates()).build();
+    }
+
+    @GET
+    @Path("/updates/types")
+    public Response getUpdatesTypes() {
+        GraphUpdaterManager updaterManager = router.graph.updaterManager;
+        if (updaterManager == null) {
+            return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build();
+        }
+        return Response.status(Response.Status.OK).entity(updaterManager.getUpdatesTypes()).build();
+    }
+
+    //TODO
+    @GET
+    @Path("/updates/applied")
+    public Response getApplied() {
+        GraphUpdaterManager updaterManager = router.graph.updaterManager;
+        if (updaterManager == null) {
+            return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build();
+        }
+        return Response.status(Response.Status.OK).entity(updaterManager.getApplied()).build();
+    }
+
+    @GET
+    @Path("/updates/errors")
+    public Response getErrors() {
+        GraphUpdaterManager updaterManager = router.graph.updaterManager;
+        if (updaterManager == null) {
+            return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build();
+        }
+        return Response.status(Response.Status.OK).entity(updaterManager.getErrors()).build();
+    }
+
+    @GET
+    @Path("/updates/received")
+    public Response getReceived() {
+        GraphUpdaterManager updaterManager = router.graph.updaterManager;
+        if (updaterManager == null) {
+            return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build();
+        }
+        return Response.status(Response.Status.OK).entity(updaterManager.getReceived()).build();
+    }
+
+    @GET
+    @Path("/updates/received/last")
+    public Response getLastReceived() {
+        GraphUpdaterManager updaterManager = router.graph.updaterManager;
+        if (updaterManager == null) {
+            return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build();
+        }
+        return Response.status(Response.Status.OK).entity(updaterManager.getLastReceived()).build();
+    }
+
+    @GET
+    @Path("/updates/updated/last")
+    public Response getLastUpdated() {
+        GraphUpdaterManager updaterManager = router.graph.updaterManager;
+        if (updaterManager == null) {
+            return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build();
+        }
+        return Response.status(Response.Status.OK).entity(updaterManager.getLastApplied()).build();
+    }
+
+    //TODO
+    @GET
+    @Path("/feed/{feedId}/trip/{tripId}")
+    public Response getUpdatesPerFeed (@PathParam("feedId") int feedId, @PathParam ("tripId") int tripId) {
+        GraphUpdaterManager updaterManager = router.graph.updaterManager;
+        if (updaterManager == null) {
+            return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build();
+        }
+        return Response.status(Response.Status.OK).entity(updaterManager.getTypeAppliedPerFeedPerTrip(feedId, tripId)).build();
+    }
+
+    //TODO
+    @GET
+    @Path("applied/feed/{feedId}")
+    public Response getAppliedPerFeed(@PathParam("feedId") int feedId) {
+        GraphUpdaterManager updaterManager = router.graph.updaterManager;
+        if (updaterManager == null) {
+            return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build();
+        }
+        return Response.status(Response.Status.OK).entity(updaterManager.getAppliedPerFeed(feedId)).build();
+    }
+
+
+    @GET
+    @Path("applied/{lastMinutes}")
+    public Response getAppliedLastMinutes(@PathParam("lastMinutes") int minutes) {
+        GraphUpdaterManager updaterManager = router.graph.updaterManager;
+        if (updaterManager == null) {
+            return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build();
+        }
+        return Response.status(Response.Status.OK).entity(updaterManager.getAppliedLastMinutes(minutes)).build();
+    }
+
+
+
+
+
 }
diff --git a/src/main/java/org/opentripplanner/routing/graph/Graph.java b/src/main/java/org/opentripplanner/routing/graph/Graph.java
index 755d877a239..73f2066d280 100644
--- a/src/main/java/org/opentripplanner/routing/graph/Graph.java
+++ b/src/main/java/org/opentripplanner/routing/graph/Graph.java
@@ -116,6 +116,8 @@ public class Graph implements Serializable {
 
     public final Deduplicator deduplicator = new Deduplicator();
 
+    private static Set correctStreamAddresses = new HashSet<>();
+
     /**
      * Map from GTFS ServiceIds to integers close to 0. Allows using BitSets instead of Set.
      * An empty Map is created before the Graph is built to allow registering IDs from multiple feeds.   
@@ -1033,7 +1035,7 @@ public SampleFactory getSampleFactory() {
      *
      * This speeds up calculation, but problem is that median needs to have all of latitudes/longitudes
      * in memory, this can become problematic in large installations. It works without a problem on New York State.
-     * @see GraphEnvelope
+     * //@see GraphEnvelope
      */
     public void calculateTransitCenter() {
         if (hasTransit) {
@@ -1070,4 +1072,12 @@ public long getTransitServiceStarts() {
     public long getTransitServiceEnds() {
         return transitServiceEnds;
     }
-}
+
+    public Set getCorrectStreamAddresses() {
+        return correctStreamAddresses;
+    }
+
+    public void addCorrectStreamAddress(String correctStreamAddress) {
+        correctStreamAddresses.add(correctStreamAddress);
+    }
+}
\ No newline at end of file
diff --git a/src/main/java/org/opentripplanner/standalone/OTPMain.java b/src/main/java/org/opentripplanner/standalone/OTPMain.java
index 35c595afeb9..73a223e1418 100644
--- a/src/main/java/org/opentripplanner/standalone/OTPMain.java
+++ b/src/main/java/org/opentripplanner/standalone/OTPMain.java
@@ -114,7 +114,7 @@ public void run() {
                     Graph graph = graphBuilder.getGraph();
                     graph.index(new DefaultStreetVertexIndexFactory());
                     // FIXME set true router IDs
-                    graphService.registerGraph("", new MemoryGraphSource("", graph));
+                    graphService.registerGraph("1", new MemoryGraphSource("1", graph));
                 }
             } else {
                 LOG.error("An error occurred while building the graph. Exiting.");
diff --git a/src/main/java/org/opentripplanner/updater/GraphUpdaterManager.java b/src/main/java/org/opentripplanner/updater/GraphUpdaterManager.java
index 211e3a88304..bfcce6e1275 100644
--- a/src/main/java/org/opentripplanner/updater/GraphUpdaterManager.java
+++ b/src/main/java/org/opentripplanner/updater/GraphUpdaterManager.java
@@ -13,12 +13,14 @@ the License, or (at your option) any later version.
 
 package org.opentripplanner.updater;
 
-import com.beust.jcommander.internal.Lists;
+import com.amazonaws.util.json.JSONException;
+import com.amazonaws.util.json.JSONObject;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Maps;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+
+import java.util.*;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -28,34 +30,35 @@ the License, or (at your option) any later version.
 import java.util.concurrent.TimeUnit;
 
 import org.opentripplanner.routing.graph.Graph;
+import org.opentripplanner.updater.stoptime.TimetableSnapshotSource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * This class is attached to the graph:
- * 
+ *
  * 
  * GraphUpdaterManager updaterManager = graph.getUpdaterManager();
  * 
- * + * * Each updater will run in its own thread. When changes to the graph have to be made by these * updaters, this should be done via the execute method of this manager to prevent race conditions * between graph write operations. - * + * */ public class GraphUpdaterManager { private static Logger LOG = LoggerFactory.getLogger(GraphUpdaterManager.class); - + /** * Text used for naming threads when the graph lacks a routerId. */ private static String DEFAULT_ROUTER_ID = "(default)"; - + /** * Thread factory used to create new threads. */ - + private ThreadFactory threadFactory; /** @@ -80,22 +83,25 @@ public class GraphUpdaterManager { * Parent graph of this manager */ Graph graph; + TimetableSnapshotSource snapshot; + ObjectMapper mapper = new ObjectMapper(); /** * Constructor - * + * * @param graph is parent graph of manager */ public GraphUpdaterManager(Graph graph) { this.graph = graph; - + String routerId = graph.routerId; if(routerId == null || routerId.isEmpty()) routerId = DEFAULT_ROUTER_ID; - + threadFactory = new ThreadFactoryBuilder().setNameFormat("GraphUpdater-" + routerId + "-%d").build(); scheduler = Executors.newSingleThreadScheduledExecutor(threadFactory); updaterPool = Executors.newCachedThreadPool(threadFactory); + } public void stop() { @@ -134,7 +140,7 @@ public void stop() { /** * Adds an updater to the manager and runs it immediately in its own thread. - * + * * @param updater is the updater to add and run */ public void addUpdater(final GraphUpdater updater) { @@ -160,7 +166,7 @@ public void run() { * This is the method to use to modify the graph from the updaters. The runnables will be * scheduled after each other, guaranteeing that only one of these runnables will be active at * any time. - * + * * @param runnable is a graph writer runnable */ public void execute(GraphWriterRunnable runnable) { @@ -171,11 +177,11 @@ public void execute(GraphWriterRunnable runnable) { * This is another method to use to modify the graph from the updaters. It behaves like execute, * but blocks until the runnable has been executed. This might be particularly useful in the * setup method of an updater. - * + * * @param runnable is a graph writer runnable * @throws ExecutionException * @throws InterruptedException - * @see GraphUpdaterManager.execute + * //@see GraphUpdaterManager.execute */ public void executeBlocking(GraphWriterRunnable runnable) throws InterruptedException, ExecutionException { @@ -186,7 +192,7 @@ public void executeBlocking(GraphWriterRunnable runnable) throws InterruptedExce } private Future executeReturningFuture(final GraphWriterRunnable runnable) { - // TODO: check for high water mark? + // TODO: check for high water mark? kada ostane mnogo praznih polja posle brisanja 14841 Future future = scheduler.submit(new Runnable() { @Override public void run() { @@ -203,27 +209,228 @@ public void run() { public int size() { return updaterList.size(); + } /** * Just an example of fetching status information from the graph updater manager to expose it in a web service. * More useful stuff should be added later. + * @return All stream addresses */ - public Map getUpdaterDescriptions () { - Map ret = Maps.newTreeMap(); + private Map getAllStreamAddresses() { + Map ret = new HashMap<>(); int i = 0; + for (GraphUpdater updater : updaterList) { ret.put(i++, updater.toString()); } return ret; } + /** + * @return correct and all stream addresses + */ + public String getStreamAddresses() { + try { + Map combined = new HashMap<>(); + combined.put("Correct stream addresses", graph.getCorrectStreamAddresses()); + combined.put("All stream addresses", getAllStreamAddresses()); + return mapper.writeValueAsString(combined); + } catch (JsonProcessingException e) { + e.printStackTrace(); + return "Something went wrong with Json."; + } + } + + /** + * + * @return most of the important data + */ + //TODO 14841 Extract ObjectMapper related code to one method + public String getDescription() { + TimetableSnapshotSource snap = graph.timetableSnapshotSource; + try { + return mapper.writeValueAsString(snap); + } catch (JsonProcessingException e) { + e.printStackTrace(); + return "Something went wrong with Json."; + } + } + + /** * Just an example of fetching status information from the graph updater manager to expose it in a web service. * More useful stuff should be added later. */ - public GraphUpdater getUpdater (int id) { - if (id >= updaterList.size()) return null; - return updaterList.get(id); + public String getUpdater(int id) { + String ret = "Updater does not exist."; + if (id < updaterList.size()) + ret = updaterList.get(id).toString(); + try { + return mapper.writeValueAsString(ret); + } catch (JsonProcessingException e) { + e.printStackTrace(); + return "Something went wrong with Json."; + } + } + + /** + * @param id of GraphUpdater whose type is required + * @return type of GraphUpdater + */ + public String getType(int id) { + GraphUpdater updater = null; + if (id < updaterList.size()) + updater = updaterList.get(id); + + try { + return mapper.writeValueAsString(updater==null?"No updater.":updater.getClass().toString()); + } catch (JsonProcessingException e) { + e.printStackTrace(); + return "Something went wrong with Json."; + } + } + + /** + * @return all types of updaters + */ + public String getAllTypes() { + HashMap retVal = new HashMap<>(); + int i = 0; + for (GraphUpdater up : updaterList) + retVal.put(i++, up.getClass().getName()); + try { + return mapper.writeValueAsString(retVal); + } catch (JsonProcessingException e) { + e.printStackTrace(); + return "Something went wrong with Json."; + } + } + + + /** + * @return all received updates + */ + public String getReceived() { + TimetableSnapshotSource snap = graph.timetableSnapshotSource; + try { + return mapper.writeValueAsString(snap.getReceived()); + } catch (JsonProcessingException e) { + e.printStackTrace(); + return "Something went wrong with Json."; + } + } + + /** + * @return all applied updates + */ + + public String getApplied() { + TimetableSnapshotSource snap = graph.timetableSnapshotSource; + try { + return mapper.writeValueAsString(snap.getApplied()); + } catch (JsonProcessingException e) { + e.printStackTrace(); + return "Something went wrong with Json."; + } + } + + /** + * @return all non-applied updates + */ + + public String getErrors() { + TimetableSnapshotSource snap = graph.timetableSnapshotSource; + try { + return mapper.writeValueAsString(snap.getErrors()); + } catch (JsonProcessingException e) { + e.printStackTrace(); + return "Something went wrong with Json."; + } + } + + public String getUpdatesTypes() { + TimetableSnapshotSource snap = graph.timetableSnapshotSource; + try { + return mapper.writeValueAsString(snap.getTypes()); + } catch (JsonProcessingException e) { + e.printStackTrace(); + return "Something went wrong with Json."; + } + + } + + //TODO 14841 String representation of time + public String getLastApplied() { + TimetableSnapshotSource snap = graph.timetableSnapshotSource; + try { + return mapper.writeValueAsString(snap.getLastUpdate()); + } catch (JsonProcessingException e) { + e.printStackTrace(); + return "Something went wrong with Json."; + } + + } + + //TODO 14841 + public String getLastReceived() { + + TimetableSnapshotSource snap = graph.timetableSnapshotSource; + try { + return mapper.writeValueAsString(snap.getLastReceived()); + } catch (JsonProcessingException e) { + e.printStackTrace(); + return "Something went wrong with Json."; + } + + } + + + //TODO 14841 + public List getAppliedPerFeed(int feedId) { + return null; + } + + //TODO 14841 what is tripId + public List getTypeAppliedPerFeedPerTrip(int feedId, int tripId) { + return null; + } + + //TODO 14841 what is stream + public List getTotalPerStream() { + return null; + } + + //TODO 14841 think of something + public List getSomethingNew() { + + + return null; + } + + //TODO 14841 + public String getUpdates() { + return null; + } + + + //TODO 14841 + public String getAgency(String feedId) { + try { + return mapper.writeValueAsString(graph.getAgencies(feedId)); + } catch (JsonProcessingException e) { + e.printStackTrace(); + return "Something went wrong with Json."; + } + } + + public String getAppliedLastMinutes(int minutes) { + TimetableSnapshotSource snap = graph.timetableSnapshotSource; + try { + return mapper.writeValueAsString(snap.getAppliedLastMinutes(minutes)); + } catch (JsonProcessingException e) { + e.printStackTrace(); + return "Something went wrong with Json."; + } } } diff --git a/src/main/java/org/opentripplanner/updater/stoptime/GtfsRealtimeHttpTripUpdateSource.java b/src/main/java/org/opentripplanner/updater/stoptime/GtfsRealtimeHttpTripUpdateSource.java index 9eed8dbb695..7c3e8f98ff3 100644 --- a/src/main/java/org/opentripplanner/updater/stoptime/GtfsRealtimeHttpTripUpdateSource.java +++ b/src/main/java/org/opentripplanner/updater/stoptime/GtfsRealtimeHttpTripUpdateSource.java @@ -46,6 +46,7 @@ public class GtfsRealtimeHttpTripUpdateSource implements TripUpdateSource, JsonC private String feedId; private String url; + Graph graph; @Override public void configure(Graph graph, JsonNode config) throws Exception { @@ -53,6 +54,7 @@ public void configure(Graph graph, JsonNode config) throws Exception { if (url == null) { throw new IllegalArgumentException("Missing mandatory 'url' parameter"); } + this.graph = graph; this.url = url; this.feedId = config.path("feedId").asText(); } @@ -69,7 +71,7 @@ public List getUpdates() { // Decode message feedMessage = FeedMessage.PARSER.parseFrom(is); feedEntityList = feedMessage.getEntityList(); - + // Change fullDataset value if this is an incremental update if (feedMessage.hasHeader() && feedMessage.getHeader().hasIncrementality() @@ -77,7 +79,8 @@ public List getUpdates() { .equals(GtfsRealtime.FeedHeader.Incrementality.DIFFERENTIAL)) { fullDataset = false; } - + graph.addCorrectStreamAddress(url); + // Create List of TripUpdates updates = new ArrayList(feedEntityList.size()); for (FeedEntity feedEntity : feedEntityList) { @@ -86,6 +89,7 @@ public List getUpdates() { } } catch (Exception e) { LOG.warn("Failed to parse gtfs-rt feed from " + url + ":", e); + graph.timetableSnapshotSource.addError("Failed to parse gtfs-rt feed from " + url); } return updates; } diff --git a/src/main/java/org/opentripplanner/updater/stoptime/TimetableSnapshotSource.java b/src/main/java/org/opentripplanner/updater/stoptime/TimetableSnapshotSource.java index 196ee8eebff..acc66f241de 100644 --- a/src/main/java/org/opentripplanner/updater/stoptime/TimetableSnapshotSource.java +++ b/src/main/java/org/opentripplanner/updater/stoptime/TimetableSnapshotSource.java @@ -13,16 +13,25 @@ the License, or (at your option) any later version. package org.opentripplanner.updater.stoptime; +import java.sql.Timestamp; import java.text.ParseException; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.Calendar; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; +import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; - +import java.util.stream.Collector; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.lucene.util.fst.PairOutputs; +import org.geotools.xml.xsi.XSISimpleTypes; +import org.joda.time.DateTime; import org.onebusaway.gtfs.model.Agency; import org.onebusaway.gtfs.model.AgencyAndId; import org.onebusaway.gtfs.model.Route; @@ -43,7 +52,6 @@ the License, or (at your option) any later version. import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; import com.google.transit.realtime.GtfsRealtime.TripDescriptor; import com.google.transit.realtime.GtfsRealtime.TripUpdate; import com.google.transit.realtime.GtfsRealtime.TripUpdate.StopTimeUpdate; @@ -66,6 +74,7 @@ public class TimetableSnapshotSource { */ private static final long MAX_ARRIVAL_DEPARTURE_TIME = 48 * 60 * 60; + @JsonIgnore public int logFrequency = 2000; private int appliedBlockCount = 0; @@ -75,6 +84,7 @@ public class TimetableSnapshotSource { * snapshot, just return the same one. Throttles the potentially resource-consuming task of * duplicating a TripPattern -> Timetable map and indexing the new Timetables. */ + @JsonIgnore public int maxSnapshotFrequency = 1000; // msec /** @@ -100,7 +110,10 @@ public class TimetableSnapshotSource { */ private final TripPatternCache tripPatternCache = new TripPatternCache(); - /** Should expired realtime data be purged from the graph. */ + /** + * Should expired realtime data be purged from the graph. + */ + @JsonIgnore public boolean purgeExpiredData = true; protected ServiceDate lastPurgeDate = null; @@ -113,8 +126,27 @@ public class TimetableSnapshotSource { private final Agency dummyAgency; + @JsonIgnore public GtfsRealtimeFuzzyTripMatcher fuzzyTripMatcher; + + private final List errorList = new ArrayList<>(); + private final List appliedList = new ArrayList<>(); + private final List receivedList = new ArrayList(); + private final List typeList = new ArrayList<>(); + private final List appliedSpecList = new ArrayList<>(); + + // public static Map appliedSpecMap = new HashMap<>(); + public static Map errorMap = new HashMap<>(); + public static Map appliedMap = new HashMap<>(); + public static Map receivedMap = new HashMap<>(); + public static Map typeMap = new HashMap<>(); + + private static Timestamp lastUpdate; + private static Timestamp lastReceived; + + private Runnable runnable; + public TimetableSnapshotSource(final Graph graph) { timeZone = graph.getTimeZone(); graphIndex = graph.index; @@ -123,14 +155,16 @@ public TimetableSnapshotSource(final Graph graph) { dummyAgency = new Agency(); dummyAgency.setId(""); dummyAgency.setName(""); + } /** * @return an up-to-date snapshot mapping TripPatterns to Timetables. This snapshot and the - * timetable objects it references are guaranteed to never change, so the requesting - * thread is provided a consistent view of all TripTimes. The routing thread need only - * release its reference to the snapshot to release resources. + * timetable objects it references are guaranteed to never change, so the requesting + * thread is provided a consistent view of all TripTimes. The routing thread need only + * release its reference to the snapshot to release resources. */ + @JsonIgnore public TimetableSnapshot getTimetableSnapshot() { TimetableSnapshot snapshotToReturn; @@ -170,21 +204,23 @@ private TimetableSnapshot getTimetableSnapshot(final boolean force) { /** * Method to apply a trip update list to the most recent version of the timetable snapshot. A * GTFS-RT feed is always applied against a single static feed (indicated by feedId). -<<<<<<< HEAD - * -======= - * + * <<<<<<< HEAD + *

+ * ======= + *

* However, multi-feed support is not completed and we currently assume there is only one static * feed when matching IDs. + *

+ * >>>>>>> 7296be8ffd532a13afb0bec263a9f436ab787022 * ->>>>>>> 7296be8ffd532a13afb0bec263a9f436ab787022 - * @param graph graph to update (needed for adding/changing stop patterns) + * @param graph graph to update (needed for adding/changing stop patterns) * @param fullDataset true iff the list with updates represent all updates that are active right - * now, i.e. all previous updates should be disregarded - * @param updates GTFS-RT TripUpdate's that should be applied atomically + * now, i.e. all previous updates should be disregarded + * @param updates GTFS-RT TripUpdate's that should be applied atomically * @param feedId */ public void applyTripUpdates(final Graph graph, final boolean fullDataset, final List updates, final String feedId) { + if (updates == null) { LOG.warn("updates is null"); return; @@ -192,7 +228,7 @@ public void applyTripUpdates(final Graph graph, final boolean fullDataset, final // Acquire lock on buffer bufferLock.lock(); - + lastReceived = new Timestamp(System.currentTimeMillis()); try { if (fullDataset) { // Remove all updates from the buffer @@ -201,7 +237,10 @@ public void applyTripUpdates(final Graph graph, final boolean fullDataset, final LOG.debug("message contains {} trip updates", updates.size()); int uIndex = 0; + for (TripUpdate tripUpdate : updates) { + + receivedList.add(tripUpdate.getTrip().getTripId()); if (fuzzyTripMatcher != null && tripUpdate.hasTrip()) { final TripDescriptor trip = fuzzyTripMatcher.match(feedId, tripUpdate.getTrip()); tripUpdate = tripUpdate.toBuilder().setTrip(trip).build(); @@ -209,6 +248,7 @@ public void applyTripUpdates(final Graph graph, final boolean fullDataset, final if (!tripUpdate.hasTrip()) { LOG.warn("Missing TripDescriptor in gtfs-rt trip update: \n{}", tripUpdate); + errorList.add("Missing TripDescriptor in gtfs-rt trip update: " + tripUpdate); continue; } @@ -220,6 +260,7 @@ public void applyTripUpdates(final Graph graph, final boolean fullDataset, final serviceDate = ServiceDate.parseString(tripDescriptor.getStartDate()); } catch (final ParseException e) { LOG.warn("Failed to parse start date in gtfs-rt trip update: \n{}", tripUpdate); + errorList.add("Failed to parse start date in gtfs-rt trip update: " + tripUpdate); continue; } } else { @@ -256,14 +297,23 @@ public void applyTripUpdates(final Graph graph, final boolean fullDataset, final if (applied) { appliedBlockCount++; + lastUpdate = new Timestamp(System.currentTimeMillis()); + appliedList.add(tripUpdate.getTrip().getTripId()); + typeList.add(tripScheduleRelationship.toString()); + appliedSpecList.add(new AppliedEntity(tripScheduleRelationship.toString(), feedId, tripUpdate.getTrip().getTripId())); + } else { LOG.warn("Failed to apply TripUpdate."); LOG.trace(" Contents: {}", tripUpdate); + errorList.add("Failed to apply TripUpdate. ");// + tripUpdate.getTrip().getTripId() + "."); + } if (appliedBlockCount % logFrequency == 0) { LOG.info("Applied {} trip updates.", appliedBlockCount); + } + } LOG.debug("end of update message"); @@ -276,12 +326,125 @@ public void applyTripUpdates(final Graph graph, final boolean fullDataset, final } else { getTimetableSnapshot(false); } + + if (runnable == null) { + runnable = new Runnable() { + public void run() { + LOG.info("Updating Maps."); + + errorMap = mergeMaps(toMap(errorList), errorMap); + appliedMap = mergeMaps(toMap(appliedList), appliedMap); + receivedMap = mergeMaps(toMap(receivedList), receivedMap); + typeMap = mergeMaps(toMap(typeList), typeMap); + + removeExpired(appliedSpecList, 60); + + errorList.clear(); + appliedList.clear(); + typeList.clear(); + receivedList.clear(); + + } + }; + ScheduledExecutorService service = Executors + .newSingleThreadScheduledExecutor(); + service.scheduleWithFixedDelay(runnable, 1, 1, TimeUnit.MINUTES); + } + + } finally { // Always release lock bufferLock.unlock(); } } + /** + * @param list that represents errors or updates + * @return Map which will group the same messages, and count the same occurrences + */ + static public Map toMap(List list) { + if (list == null || list.size() == 0) + return null; + return list.stream().collect(Collectors.groupingBy(s -> s, Collectors.counting())); + } + + static public Map>> toMap2(List list) { + if (list == null || list.size() == 0) + return null; + return list.stream().collect(Collectors.groupingBy(AppliedEntity::getType, Collectors.groupingBy(AppliedEntity::getFeedTrip, Collectors.toSet()))); + } + + static public Map> toMap2Number(List list) { + if (list == null || list.size() == 0) + return null; + return list.stream().collect(Collectors.groupingBy(AppliedEntity::getType, Collectors.groupingBy(AppliedEntity::getFeedTrip, Collectors.counting()))); + } + + static public Map toMap(List list, String type) { + + if (list == null || list.size() == 0) + return null; + switch (type) { + case "feedId": + return list.stream().collect(Collectors.groupingBy(AppliedEntity::getFeedId, Collectors.counting())); + case "tripId": + return list.stream().collect(Collectors.groupingBy(AppliedEntity::getTripId, Collectors.counting())); + case "type": + return list.stream().collect(Collectors.groupingBy(AppliedEntity::getType, Collectors.counting())); + default: + LOG.error("Wrong parameter for Generating Map."); + return null; + } + } + + /** + * @param firstMap + * @param secondMap + * @return Map which will group the same messages, and count the same occurrences + */ + static public Map mergeMaps(Map firstMap, Map secondMap) { + if (firstMap == null) + return secondMap; + else if (secondMap == null) + return firstMap; + Map result = Stream.concat(firstMap.entrySet().stream(), secondMap.entrySet().stream()) + .collect(Collectors.toMap( + entry -> entry.getKey(), // The key + entry -> entry.getValue(), // The value + // The "merger" as a method reference + (a, b) -> a + b + ) + ); + return result; + } + + static void removeExpired(List c, long minutes) { + if (c == null) + return; + Timestamp expirationDate = new Timestamp(System.currentTimeMillis() - minutes * 1000 * 60); + + for (Iterator it = c.iterator(); it.hasNext(); ) + if (((AppliedEntity) it.next()).getTimestamp().before(expirationDate)) + it.remove(); + + } + + @JsonIgnore + private List getSelected(int minutes){ + if (appliedSpecList == null) + return null; + ArrayList c = new ArrayList(appliedSpecList); + + Timestamp expirationDate = new Timestamp(System.currentTimeMillis() - minutes * 1000 * 60); + + for (Iterator it = c.iterator(); it.hasNext(); ) + if (((AppliedEntity) it.next()).getTimestamp().before(expirationDate)) + it.remove(); + + return c; + } + + /** * Determine how the trip update should be handled. * @@ -306,7 +469,7 @@ private TripDescriptor.ScheduleRelationship determineTripScheduleRelationship(fi final StopTimeUpdate.ScheduleRelationship stopScheduleRelationship = stopTimeUpdate .getScheduleRelationship(); if (stopScheduleRelationship.equals(StopTimeUpdate.ScheduleRelationship.SKIPPED) - // TODO: uncomment next line when StopTimeUpdate.ScheduleRelationship.ADDED exists + // TODO: uncomment next line when StopTimeUpdate.ScheduleRelationship.ADDED exists // || stopScheduleRelationship.equals(StopTimeUpdate.ScheduleRelationship.ADDED) ) { hasModifiedStops = true; @@ -364,7 +527,7 @@ private boolean handleScheduledTrip(final TripUpdate tripUpdate, final String fe * @return true iff successful */ private boolean validateAndHandleAddedTrip(final Graph graph, final TripUpdate tripUpdate, - final String feedId, final ServiceDate serviceDate) { + final String feedId, final ServiceDate serviceDate) { // Preconditions Preconditions.checkNotNull(graph); Preconditions.checkNotNull(tripUpdate); @@ -552,7 +715,7 @@ private boolean isStopSkipped(final StopTimeUpdate stopTimeUpdate) { * @return true iff successful */ private boolean handleAddedTrip(final Graph graph, final TripUpdate tripUpdate, final List stops, - final String feedId, final ServiceDate serviceDate) { + final String feedId, final ServiceDate serviceDate) { // Preconditions Preconditions.checkNotNull(stops); Preconditions.checkArgument(tripUpdate.getStopTimeUpdateCount() == stops.size(), @@ -623,8 +786,8 @@ private boolean handleAddedTrip(final Graph graph, final TripUpdate tripUpdate, * @return true iff successful */ private boolean addTripToGraphAndBuffer(final String feedId, final Graph graph, final Trip trip, - final TripUpdate tripUpdate, final List stops, final ServiceDate serviceDate, - final RealTimeState realTimeState) { + final TripUpdate tripUpdate, final List stops, final ServiceDate serviceDate, + final RealTimeState realTimeState) { // Preconditions Preconditions.checkNotNull(stops); @@ -740,7 +903,7 @@ private boolean addTripToGraphAndBuffer(final String feedId, final Graph graph, */ private boolean cancelScheduledTrip(String feedId, String tripId, final ServiceDate serviceDate) { boolean success = false; - + final TripPattern pattern = getPatternForTripId(feedId, tripId); if (pattern != null) { @@ -879,7 +1042,7 @@ private boolean validateAndHandleModifiedTrip(final Graph graph, final TripUpdat * @return true iff successful */ private boolean handleModifiedTrip(final Graph graph, final Trip trip, final TripUpdate tripUpdate, final List stops, - final String feedId, final ServiceDate serviceDate) { + final String feedId, final ServiceDate serviceDate) { // Preconditions Preconditions.checkNotNull(stops); Preconditions.checkArgument(tripUpdate.getStopTimeUpdateCount() == stops.size(), @@ -938,7 +1101,7 @@ private boolean purgeExpiredData() { } /** - * Retrieve a trip pattern given a feed id and trid id. + * Retrieve a trip pattern given a feed id and trip id. * * @param feedId feed id for the trip id * @param tripId trip id without agency @@ -986,4 +1149,175 @@ private Stop getStopForStopId(String feedId, String stopId) { return stop; } + + public Map getErrors() { + return errorMap; + } + + public Map getApplied() { + return appliedMap; + } + + public Map getReceived() { + return receivedMap; + } + + public Map getTypes() { + return typeMap; + } + + public Map getAppliedPerFeed() { + return toMap(appliedSpecList, "feedId"); + } + + public Map getAppliedPerTrip() { + return toMap(appliedSpecList, "tripId"); + } + + public Map getAppliedPerType() { + return toMap(appliedSpecList, "type"); + } + + public Map>> getAppliedTypePerFeedPerTrip() { + return toMap2(appliedSpecList); + } + + public String getLastUpdate() { + return lastUpdate.toString(); + } + + public String getLastReceived() { + return lastReceived.toString(); + } + + public Map> getAppliedTypePerFeedPerTripNumber() { + return toMap2Number(appliedSpecList); + } + + public void addError(String err) { + errorList.add(err); + } + @JsonIgnore + public List getAppliedLastMinutes(int minutes) { + + return getSelected(minutes); + + } + +} + +class AppliedEntity implements Comparable { + @JsonIgnore + String type; + String feedId; + String tripId; + @JsonIgnore + FeedTrip feedTrip; + @JsonIgnore + Timestamp timestamp; + + AppliedEntity(String type, + String feedId, + String tripId) { + this.type = type; + this.feedId = feedId; + this.tripId = tripId; + this.feedTrip = new FeedTrip(feedId, tripId); + this.timestamp = new Timestamp(System.currentTimeMillis()); + } + + public Timestamp getTimestamp() { + return timestamp; + } + + public String getTripId() { + return tripId; + } + + public String getType() { + return type; + } + + public String getFeedId() { + return feedId; + } + + public FeedTrip getFeedTrip() { + return feedTrip; + } + + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (o == null) { + return false; + } + if (!(o instanceof AppliedEntity)) { + return false; + } + AppliedEntity ft = (AppliedEntity) o; + return ft.getFeedId().equals(this.getFeedId()) && ft.getTripId().equals(this.getTripId()) && ft.getType().equals(this.getType()); + } + + @Override + public int hashCode() { + return this.getFeedId().hashCode() * 13 + this.getTripId().hashCode() * 17 + this.getType().hashCode() * 31; + } + + @Override + public int compareTo(Object T1) { + + + return 1; + } + + +} + +class FeedTrip { + String feedId; + String tripId; + + public FeedTrip(String feedId, String tripId) { + this.feedId = feedId; + this.tripId = tripId; + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (o == null) { + return false; + } + if (!(o instanceof FeedTrip)) { + return false; + } + + FeedTrip ft = (FeedTrip) o; + return ft.getFeedId().equals(this.getFeedId()) && ft.getTripId().equals(this.getTripId()); + } + + @Override + public int hashCode() { + return this.getFeedId().hashCode() * 13 + this.getTripId().hashCode() * 17; + } + + + public String getFeedId() { + return feedId; + } + + public String getTripId() { + return tripId; + } + + @Override + public String toString() { + return "feedTrip"; + } } diff --git a/src/main/java/org/opentripplanner/updater/stoptime/TripUpdateGraphWriterRunnable.java b/src/main/java/org/opentripplanner/updater/stoptime/TripUpdateGraphWriterRunnable.java index 5db3b4168fe..4a93c0e6879 100644 --- a/src/main/java/org/opentripplanner/updater/stoptime/TripUpdateGraphWriterRunnable.java +++ b/src/main/java/org/opentripplanner/updater/stoptime/TripUpdateGraphWriterRunnable.java @@ -13,6 +13,7 @@ the License, or (at your option) any later version. package org.opentripplanner.updater.stoptime; +import java.util.ArrayList; import java.util.List; import org.opentripplanner.routing.graph.Graph; @@ -40,6 +41,7 @@ public class TripUpdateGraphWriterRunnable implements GraphWriterRunnable { private final String feedId; + public TripUpdateGraphWriterRunnable(final boolean fullDataset, final List updates, final String feedId) { // Preconditions Preconditions.checkNotNull(updates); @@ -49,6 +51,7 @@ public TripUpdateGraphWriterRunnable(final boolean fullDataset, final List(feedEntityList.size()); for (FeedEntity feedEntity : feedEntityList) { diff --git a/src/scripts/otp b/src/scripts/otp old mode 100755 new mode 100644 diff --git a/src/scripts/otp-batch-analyst b/src/scripts/otp-batch-analyst old mode 100755 new mode 100644 diff --git a/src/test/resources/generateGtfs.py b/src/test/resources/generateGtfs.py old mode 100755 new mode 100644 diff --git a/src/test/resources/portland/Streets_pdx.shx b/src/test/resources/portland/Streets_pdx.shx old mode 100755 new mode 100644 From 9b4752bb79b83ec4e577e762e0585e93bf9458eb Mon Sep 17 00:00:00 2001 From: Milena Mandic Date: Thu, 26 Jan 2017 02:35:30 +0100 Subject: [PATCH 11/12] Web service for status on real time updates v.2 --- .../api/resource/UpdaterStatusResource.java | 139 ++++++-- .../updater/GraphUpdaterManager.java | 199 ++++------- .../stoptime/TimetableSnapshotSource.java | 333 +++++++++++++++--- 3 files changed, 465 insertions(+), 206 deletions(-) diff --git a/src/main/java/org/opentripplanner/api/resource/UpdaterStatusResource.java b/src/main/java/org/opentripplanner/api/resource/UpdaterStatusResource.java index b22e0d84377..00dd08a56a3 100644 --- a/src/main/java/org/opentripplanner/api/resource/UpdaterStatusResource.java +++ b/src/main/java/org/opentripplanner/api/resource/UpdaterStatusResource.java @@ -23,12 +23,15 @@ public class UpdaterStatusResource { private static final Logger LOG = LoggerFactory.getLogger(UpdaterStatusResource.class); - /** Choose short or long form of results. */ - @QueryParam("detail") private boolean detail = false; + /** + * Choose short or long form of results. + */ + @QueryParam("detail") + private boolean detail = false; Router router; - public UpdaterStatusResource (@Context OTPServer otpServer, @PathParam("routerId") String routerId) { + public UpdaterStatusResource(@Context OTPServer otpServer, @PathParam("routerId") String routerId) { router = otpServer.getRouter(routerId); } @@ -45,10 +48,13 @@ public Response getDescription() { return Response.status(Response.Status.OK).entity(updaterManager.getDescription()).build(); } - /** Return a list of all agencies in the graph. */ + /** + * + * @return a list of all agencies in the graph for the given feedId. + */ @GET @Path("/agency/{feedId}") - public Response getAgencies (@PathParam("feedId") String feedId) { + public Response getAgencies(@PathParam("feedId") String feedId) { GraphUpdaterManager updaterManager = router.graph.updaterManager; if (updaterManager == null) { return Response.status(Response.Status.NOT_FOUND).entity("There is no updaters running.").build(); @@ -56,11 +62,13 @@ public Response getAgencies (@PathParam("feedId") String feedId) { return Response.status(Response.Status.OK).entity(updaterManager.getAgency(feedId)).build(); } - //TODO does not look good - /** Return status for a specific updater. */ + /** + * + * @return status for a specific updater. + */ @GET @Path("/{updaterId}") - public Response getTypeOfUpdater (@PathParam("updaterId") int updaterId) { + public Response getTypeOfUpdater(@PathParam("updaterId") int updaterId) { GraphUpdaterManager updaterManager = router.graph.updaterManager; if (updaterManager == null) { return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build(); @@ -68,8 +76,10 @@ public Response getTypeOfUpdater (@PathParam("updaterId") int updaterId) { return Response.status(Response.Status.OK).entity(updaterManager.getUpdater(updaterId)).build(); } - - //TODO + /** + * + * @return correct and all stream addresses + */ @GET @Path("/stream") public Response getStreamAddresses() { @@ -80,7 +90,10 @@ public Response getStreamAddresses() { return Response.status(Response.Status.OK).entity(updaterManager.getStreamAddresses()).build(); } - + /** + * + * @return types of updater + */ @GET @Path("/types") public Response getTypes() { @@ -91,17 +104,40 @@ public Response getTypes() { return Response.status(Response.Status.OK).entity(updaterManager.getAllTypes()).build(); } - //TODO all updates + /** + * + * @param updaterId + * @return type of updater for updaterId + */ + @GET + @Path("/types/{id}") + public Response getTypePerId(@PathParam("id") int updaterId) { + GraphUpdaterManager updaterManager = router.graph.updaterManager; + if (updaterManager == null) { + return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build(); + } + return Response.status(Response.Status.OK).entity(updaterManager.getType(updaterId)).build(); + } + + /** + * + * @return all updates grouped by tripid, + * exposing the number of time tripid showed in updates + */ @GET @Path("/updates") - public Response getUpdates () { + public Response getUpdates() { GraphUpdaterManager updaterManager = router.graph.updaterManager; if (updaterManager == null) { return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build(); } - return Response.status(Response.Status.OK).entity(updaterManager.getUpdates()).build(); + return Response.status(Response.Status.OK).entity(updaterManager.getReceived()).build(); } + /** + * + * @return the number of updates per type + */ @GET @Path("/updates/types") public Response getUpdatesTypes() { @@ -112,7 +148,10 @@ public Response getUpdatesTypes() { return Response.status(Response.Status.OK).entity(updaterManager.getUpdatesTypes()).build(); } - //TODO + /** + * + * @return the number of updates applied per tripId + */ @GET @Path("/updates/applied") public Response getApplied() { @@ -123,6 +162,11 @@ public Response getApplied() { return Response.status(Response.Status.OK).entity(updaterManager.getApplied()).build(); } + /** + * + * @return the errors for updates grouped by its String representation + * exposing the number of occurrences for each error + */ @GET @Path("/updates/errors") public Response getErrors() { @@ -133,40 +177,57 @@ public Response getErrors() { return Response.status(Response.Status.OK).entity(updaterManager.getErrors()).build(); } + /** + * + * @return the errors for last block of updates + */ @GET - @Path("/updates/received") - public Response getReceived() { + @Path("/updates/errors/last") + public Response getLastErrors() { GraphUpdaterManager updaterManager = router.graph.updaterManager; if (updaterManager == null) { return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build(); } - return Response.status(Response.Status.OK).entity(updaterManager.getReceived()).build(); + return Response.status(Response.Status.OK).entity(updaterManager.getLastErrors()).build(); } + /** + * + * @return time and tripId for the last received and applied update + */ @GET - @Path("/updates/received/last") - public Response getLastReceived() { + @Path("/updates/last") + public Response getLastAppliedReceived() { GraphUpdaterManager updaterManager = router.graph.updaterManager; if (updaterManager == null) { return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build(); } - return Response.status(Response.Status.OK).entity(updaterManager.getLastReceived()).build(); + return Response.status(Response.Status.OK).entity(updaterManager.getLastAppliedReceived()).build(); } + /** + * + * @return the ratio between received and applied update + */ @GET - @Path("/updates/updated/last") - public Response getLastUpdated() { + @Path("/updates/ratio") + public Response getReceivedApplied() { GraphUpdaterManager updaterManager = router.graph.updaterManager; if (updaterManager == null) { return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build(); } - return Response.status(Response.Status.OK).entity(updaterManager.getLastApplied()).build(); + return Response.status(Response.Status.OK).entity(updaterManager.getReceivedApplied()).build(); } - //TODO + /** + * + * @param feedId + * @param tripId + * @return the number of updates for feedId and tripId grouped by type + */ @GET - @Path("/feed/{feedId}/trip/{tripId}") - public Response getUpdatesPerFeed (@PathParam("feedId") int feedId, @PathParam ("tripId") int tripId) { + @Path("/updates/applied/feed/{feedId}/trip/{tripId}") + public Response getUpdatesPerFeedPerTrip(@PathParam("feedId") String feedId, @PathParam("tripId") String tripId) { GraphUpdaterManager updaterManager = router.graph.updaterManager; if (updaterManager == null) { return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build(); @@ -174,10 +235,14 @@ public Response getUpdatesPerFeed (@PathParam("feedId") int feedId, @PathParam ( return Response.status(Response.Status.OK).entity(updaterManager.getTypeAppliedPerFeedPerTrip(feedId, tripId)).build(); } - //TODO + /** + * + * @param feedId + * @return the number of updates applied for feedId + */ @GET - @Path("applied/feed/{feedId}") - public Response getAppliedPerFeed(@PathParam("feedId") int feedId) { + @Path("/updates/applied/feed/{feedId}") + public Response getAppliedPerFeed(@PathParam("feedId") String feedId) { GraphUpdaterManager updaterManager = router.graph.updaterManager; if (updaterManager == null) { return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build(); @@ -185,9 +250,14 @@ public Response getAppliedPerFeed(@PathParam("feedId") int feedId) { return Response.status(Response.Status.OK).entity(updaterManager.getAppliedPerFeed(feedId)).build(); } - + /** + * + * @param minutes + * @return information about updates that occurred + * in the last number of minutes + */ @GET - @Path("applied/{lastMinutes}") + @Path("updates/applied/{lastMinutes}") public Response getAppliedLastMinutes(@PathParam("lastMinutes") int minutes) { GraphUpdaterManager updaterManager = router.graph.updaterManager; if (updaterManager == null) { @@ -195,9 +265,4 @@ public Response getAppliedLastMinutes(@PathParam("lastMinutes") int minutes) { } return Response.status(Response.Status.OK).entity(updaterManager.getAppliedLastMinutes(minutes)).build(); } - - - - - } diff --git a/src/main/java/org/opentripplanner/updater/GraphUpdaterManager.java b/src/main/java/org/opentripplanner/updater/GraphUpdaterManager.java index bfcce6e1275..f4bd7dfa280 100644 --- a/src/main/java/org/opentripplanner/updater/GraphUpdaterManager.java +++ b/src/main/java/org/opentripplanner/updater/GraphUpdaterManager.java @@ -17,6 +17,7 @@ the License, or (at your option) any later version. import com.amazonaws.util.json.JSONObject; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import com.google.common.collect.Maps; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -192,7 +193,7 @@ public void executeBlocking(GraphWriterRunnable runnable) throws InterruptedExce } private Future executeReturningFuture(final GraphWriterRunnable runnable) { - // TODO: check for high water mark? kada ostane mnogo praznih polja posle brisanja 14841 + // TODO: check for high water mark? Future future = scheduler.submit(new Runnable() { @Override public void run() { @@ -213,8 +214,6 @@ public int size() { } /** - * Just an example of fetching status information from the graph updater manager to expose it in a web service. - * More useful stuff should be added later. * @return All stream addresses */ private Map getAllStreamAddresses() { @@ -231,47 +230,29 @@ private Map getAllStreamAddresses() { * @return correct and all stream addresses */ public String getStreamAddresses() { - try { Map combined = new HashMap<>(); combined.put("Correct stream addresses", graph.getCorrectStreamAddresses()); combined.put("All stream addresses", getAllStreamAddresses()); - return mapper.writeValueAsString(combined); - } catch (JsonProcessingException e) { - e.printStackTrace(); - return "Something went wrong with Json."; - } + return callMapper(combined); } /** - * - * @return most of the important data + * @return most of the important statistical data */ - //TODO 14841 Extract ObjectMapper related code to one method public String getDescription() { TimetableSnapshotSource snap = graph.timetableSnapshotSource; - try { - return mapper.writeValueAsString(snap); - } catch (JsonProcessingException e) { - e.printStackTrace(); - return "Something went wrong with Json."; - } + return callMapper(snap); } - /** - * Just an example of fetching status information from the graph updater manager to expose it in a web service. - * More useful stuff should be added later. + * @param id + * @return Description of GraphUpdater for id */ public String getUpdater(int id) { String ret = "Updater does not exist."; if (id < updaterList.size()) ret = updaterList.get(id).toString(); - try { - return mapper.writeValueAsString(ret); - } catch (JsonProcessingException e) { - e.printStackTrace(); - return "Something went wrong with Json."; - } + return callMapper(ret); } /** @@ -282,13 +263,7 @@ public String getType(int id) { GraphUpdater updater = null; if (id < updaterList.size()) updater = updaterList.get(id); - - try { - return mapper.writeValueAsString(updater==null?"No updater.":updater.getClass().toString()); - } catch (JsonProcessingException e) { - e.printStackTrace(); - return "Something went wrong with Json."; - } + return callMapper(updater == null ? "No updater." : updater.getClass().toString()); } /** @@ -299,138 +274,116 @@ public String getAllTypes() { int i = 0; for (GraphUpdater up : updaterList) retVal.put(i++, up.getClass().getName()); - try { - return mapper.writeValueAsString(retVal); - } catch (JsonProcessingException e) { - e.printStackTrace(); - return "Something went wrong with Json."; - } + return callMapper(retVal); } - /** - * @return all received updates + * @return the number of all received updates per tripId */ public String getReceived() { TimetableSnapshotSource snap = graph.timetableSnapshotSource; - try { - return mapper.writeValueAsString(snap.getReceived()); - } catch (JsonProcessingException e) { - e.printStackTrace(); - return "Something went wrong with Json."; - } + return callMapper(snap.getReceived()); } /** - * @return all applied updates + * @return the number of all applied updates per tripId */ - public String getApplied() { TimetableSnapshotSource snap = graph.timetableSnapshotSource; - try { - return mapper.writeValueAsString(snap.getApplied()); - } catch (JsonProcessingException e) { - e.printStackTrace(); - return "Something went wrong with Json."; - } + return callMapper(snap.getApplied()); } /** - * @return all non-applied updates + * @return error for non applied update */ - public String getErrors() { TimetableSnapshotSource snap = graph.timetableSnapshotSource; - try { - return mapper.writeValueAsString(snap.getErrors()); - } catch (JsonProcessingException e) { - e.printStackTrace(); - return "Something went wrong with Json."; - } + return callMapper(snap.getErrors()); } - public String getUpdatesTypes() { + /** + * @return errors for last update block + */ + public String getLastErrors() { TimetableSnapshotSource snap = graph.timetableSnapshotSource; - try { - return mapper.writeValueAsString(snap.getTypes()); - } catch (JsonProcessingException e) { - e.printStackTrace(); - return "Something went wrong with Json."; - } - + return callMapper(snap.getLastErrors()); } - //TODO 14841 String representation of time - public String getLastApplied() { + /** + * @return the number of updates per each type + */ + public String getUpdatesTypes() { TimetableSnapshotSource snap = graph.timetableSnapshotSource; - try { - return mapper.writeValueAsString(snap.getLastUpdate()); - } catch (JsonProcessingException e) { - e.printStackTrace(); - return "Something went wrong with Json."; - } - + return callMapper(snap.getTypes()); } - //TODO 14841 - public String getLastReceived() { - + /** + * @return the timestamp in milliseconds for last applied and received updates and their trip id + */ + public String getLastAppliedReceived() { TimetableSnapshotSource snap = graph.timetableSnapshotSource; - try { - return mapper.writeValueAsString(snap.getLastReceived()); - } catch (JsonProcessingException e) { - e.printStackTrace(); - return "Something went wrong with Json."; - } - + return callMapper(snap.getLastAppliedReceived()); } - - //TODO 14841 - public List getAppliedPerFeed(int feedId) { - return null; - } - - //TODO 14841 what is tripId - public List getTypeAppliedPerFeedPerTrip(int feedId, int tripId) { - return null; - } - - //TODO 14841 what is stream - public List getTotalPerStream() { - return null; + /** + * @return the ratio of received and applied updates + */ + public String getReceivedApplied(){ + TimetableSnapshotSource snap = graph.timetableSnapshotSource; + return callMapper(snap.getReceivedApplied()); } - //TODO 14841 think of something - public List getSomethingNew() { - - - return null; + /** + * @param feedId + * @return the number of updates per provided feedId + * all updates older than @see TimetableSnapshotSource 60 minutes are discarded. + */ + public String getAppliedPerFeed(String feedId) { + TimetableSnapshotSource snap = graph.timetableSnapshotSource; + return callMapper(snap.getAppliedPerFeed(feedId)); } - //TODO 14841 - public String getUpdates() { - return null; + /** + * @param feedId + * @param tripId + * @return the number of updates grouped by type per feedId and triId + * all updates older than @see TimetableSnapshotSource 60 minutes are discarded. + */ + public String getTypeAppliedPerFeedPerTrip(String feedId, String tripId) { + TimetableSnapshotSource snap = graph.timetableSnapshotSource; + return callMapper(snap.getAppliedTypePerFeedPerTrip(feedId, tripId)); } - - //TODO 14841 + /** + * @param feedId + * @return the description of agency for feedId + */ public String getAgency(String feedId) { - try { - return mapper.writeValueAsString(graph.getAgencies(feedId)); - } catch (JsonProcessingException e) { - e.printStackTrace(); - return "Something went wrong with Json."; - } + return callMapper(graph.getAgencies(feedId)); } + /** + * @param minutes + * @return all updates that happened in the last number of minutes + * all updates older than @see TimetableSnapshotSource 60 minutes are discarded. + */ public String getAppliedLastMinutes(int minutes) { TimetableSnapshotSource snap = graph.timetableSnapshotSource; + return callMapper(snap.getAppliedLastMinutes(minutes)); + } + + /** + * @param o any object + * @return String representation of an Object in Json format + */ + private String callMapper(Object o) { try { - return mapper.writeValueAsString(snap.getAppliedLastMinutes(minutes)); + return mapper.writeValueAsString(o); } catch (JsonProcessingException e) { e.printStackTrace(); return "Something went wrong with Json."; } } -} + + +} \ No newline at end of file diff --git a/src/main/java/org/opentripplanner/updater/stoptime/TimetableSnapshotSource.java b/src/main/java/org/opentripplanner/updater/stoptime/TimetableSnapshotSource.java index acc66f241de..5a2a987ed6f 100644 --- a/src/main/java/org/opentripplanner/updater/stoptime/TimetableSnapshotSource.java +++ b/src/main/java/org/opentripplanner/updater/stoptime/TimetableSnapshotSource.java @@ -20,15 +20,20 @@ the License, or (at your option) any later version. import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.BiFunction; import java.util.stream.Collector; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.*; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.introspect.AnnotatedClass; +import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector; +import com.fasterxml.jackson.databind.ser.FilterProvider; +import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter; +import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider; +import com.google.common.collect.ImmutableMap; import org.apache.lucene.util.fst.PairOutputs; import org.geotools.xml.xsi.XSISimpleTypes; import org.joda.time.DateTime; @@ -56,6 +61,10 @@ the License, or (at your option) any later version. import com.google.transit.realtime.GtfsRealtime.TripUpdate; import com.google.transit.realtime.GtfsRealtime.TripUpdate.StopTimeUpdate; +import javax.ws.rs.GET; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; + /** * This class should be used to create snapshots of lookup tables of realtime data. This is * necessary to provide planning threads a consistent constant view of a graph with realtime data at @@ -136,14 +145,19 @@ public class TimetableSnapshotSource { private final List typeList = new ArrayList<>(); private final List appliedSpecList = new ArrayList<>(); - // public static Map appliedSpecMap = new HashMap<>(); public static Map errorMap = new HashMap<>(); public static Map appliedMap = new HashMap<>(); public static Map receivedMap = new HashMap<>(); public static Map typeMap = new HashMap<>(); private static Timestamp lastUpdate; + private static String lastUpdatedTrip; private static Timestamp lastReceived; + private static String lastReceivedTrip; + private static int receivedNum = 0; + private static int appliedNum = 0; + private static Set lastErrors = new HashSet<>(); + private Runnable runnable; @@ -220,7 +234,6 @@ private TimetableSnapshot getTimetableSnapshot(final boolean force) { * @param feedId */ public void applyTripUpdates(final Graph graph, final boolean fullDataset, final List updates, final String feedId) { - if (updates == null) { LOG.warn("updates is null"); return; @@ -228,7 +241,7 @@ public void applyTripUpdates(final Graph graph, final boolean fullDataset, final // Acquire lock on buffer bufferLock.lock(); - lastReceived = new Timestamp(System.currentTimeMillis()); + lastErrors.clear(); try { if (fullDataset) { // Remove all updates from the buffer @@ -239,8 +252,10 @@ public void applyTripUpdates(final Graph graph, final boolean fullDataset, final int uIndex = 0; for (TripUpdate tripUpdate : updates) { - - receivedList.add(tripUpdate.getTrip().getTripId()); + receivedNum++; + lastReceived = new Timestamp(System.currentTimeMillis()); + lastReceivedTrip = tripUpdate.getTrip().getTripId(); + receivedList.add(lastReceivedTrip); if (fuzzyTripMatcher != null && tripUpdate.hasTrip()) { final TripDescriptor trip = fuzzyTripMatcher.match(feedId, tripUpdate.getTrip()); tripUpdate = tripUpdate.toBuilder().setTrip(trip).build(); @@ -249,6 +264,7 @@ public void applyTripUpdates(final Graph graph, final boolean fullDataset, final if (!tripUpdate.hasTrip()) { LOG.warn("Missing TripDescriptor in gtfs-rt trip update: \n{}", tripUpdate); errorList.add("Missing TripDescriptor in gtfs-rt trip update: " + tripUpdate); + lastErrors.add("Missing TripDescriptor in gtfs-rt trip update"); continue; } @@ -261,6 +277,7 @@ public void applyTripUpdates(final Graph graph, final boolean fullDataset, final } catch (final ParseException e) { LOG.warn("Failed to parse start date in gtfs-rt trip update: \n{}", tripUpdate); errorList.add("Failed to parse start date in gtfs-rt trip update: " + tripUpdate); + lastErrors.add("Failed to parse start date in gtfs-rt trip update"); continue; } } else { @@ -297,16 +314,18 @@ public void applyTripUpdates(final Graph graph, final boolean fullDataset, final if (applied) { appliedBlockCount++; + appliedNum++; lastUpdate = new Timestamp(System.currentTimeMillis()); - appliedList.add(tripUpdate.getTrip().getTripId()); + lastUpdatedTrip = tripUpdate.getTrip().getTripId(); + appliedList.add(lastUpdatedTrip); typeList.add(tripScheduleRelationship.toString()); - appliedSpecList.add(new AppliedEntity(tripScheduleRelationship.toString(), feedId, tripUpdate.getTrip().getTripId())); + appliedSpecList.add(new AppliedEntity(tripScheduleRelationship.toString(), feedId, lastUpdatedTrip)); } else { LOG.warn("Failed to apply TripUpdate."); LOG.trace(" Contents: {}", tripUpdate); errorList.add("Failed to apply TripUpdate. ");// + tripUpdate.getTrip().getTripId() + "."); - + lastErrors.add("Failed to apply TripUpdate."); } if (appliedBlockCount % logFrequency == 0) { @@ -326,7 +345,9 @@ public void applyTripUpdates(final Graph graph, final boolean fullDataset, final } else { getTimetableSnapshot(false); } - + /** + * Every minute Maps for update statistics are updated + */ if (runnable == null) { runnable = new Runnable() { public void run() { @@ -362,47 +383,91 @@ public void run() { * @param list that represents errors or updates * @return Map which will group the same messages, and count the same occurrences */ - static public Map toMap(List list) { + static private Map toMap(List list) { if (list == null || list.size() == 0) return null; return list.stream().collect(Collectors.groupingBy(s -> s, Collectors.counting())); } - static public Map>> toMap2(List list) { + /** + * + * @param list + * @return map grouped by occurrences of entry in the list + */ + static private Map toMapObject(List list) { if (list == null || list.size() == 0) return null; - return list.stream().collect(Collectors.groupingBy(AppliedEntity::getType, Collectors.groupingBy(AppliedEntity::getFeedTrip, Collectors.toSet()))); + return list.stream().collect(Collectors.groupingBy(s -> s, Collectors.counting())); } - static public Map> toMap2Number(List list) { + /** + * + * @param list + * @return map of applied updates groupedby type, feedid and tripid + */ + static private Map>> toMap2(List list) { if (list == null || list.size() == 0) return null; - return list.stream().collect(Collectors.groupingBy(AppliedEntity::getType, Collectors.groupingBy(AppliedEntity::getFeedTrip, Collectors.counting()))); + return list.stream().collect(Collectors.groupingBy(AppliedEntity::getType, Collectors.groupingBy(AppliedEntity::getFeedId, Collectors.groupingBy(AppliedEntity::getTripId, Collectors.counting())))); + } - static public Map toMap(List list, String type) { + /** + * + * @param list + * @param feedId + * @param tripId + * @return map formed from a list, grouped by occurrences of types per specific feedId and tripId + */ + static private Map toMap2(List list, String feedId, String tripId) { + if (feedId == null || tripId == null) { + return null; + } + FeedTrip ft = new FeedTrip(feedId, tripId); + BiFunction, List> byFeedTrip = + (feedtrip, l) -> l.stream() + .filter(a -> !(a.getFeedTrip().equals(feedtrip))) + .collect(Collectors.toList()); + return byFeedTrip.apply(ft, list).stream().collect(Collectors.groupingBy(AppliedEntity::getType, Collectors.counting())); + } + /** + * + * @param list + * @param type + * @param id + * @return Object which is map formed from a list grouped by type if id is null, or + * if id not null the value of entry with key id + */ + static private Object toMap(List list, String type, String id) { if (list == null || list.size() == 0) return null; + Map ret; switch (type) { case "feedId": - return list.stream().collect(Collectors.groupingBy(AppliedEntity::getFeedId, Collectors.counting())); + ret = list.stream().collect(Collectors.groupingBy(AppliedEntity::getFeedId, Collectors.counting())); + break; case "tripId": - return list.stream().collect(Collectors.groupingBy(AppliedEntity::getTripId, Collectors.counting())); + ret = list.stream().collect(Collectors.groupingBy(AppliedEntity::getTripId, Collectors.counting())); + break; case "type": - return list.stream().collect(Collectors.groupingBy(AppliedEntity::getType, Collectors.counting())); + ret = list.stream().collect(Collectors.groupingBy(AppliedEntity::getType, Collectors.counting())); + break; default: LOG.error("Wrong parameter for Generating Map."); return null; } + if (id == null) + return ret; + return ret.get(id); } /** * @param firstMap * @param secondMap - * @return Map which will group the same messages, and count the same occurrences + * @return Map that is concatenation of two maps grouped by the same messages - key, and count the same occurrences -value in the map */ - static public Map mergeMaps(Map firstMap, Map secondMap) { + static private Map mergeMaps(Map firstMap, Map secondMap) { if (firstMap == null) return secondMap; else if (secondMap == null) @@ -418,6 +483,11 @@ else if (secondMap == null) return result; } + /** + * removes all older than number of minutes data from list c + * @param c + * @param minutes + */ static void removeExpired(List c, long minutes) { if (c == null) return; @@ -429,15 +499,20 @@ static void removeExpired(List c, long minutes) { } + /** + * + * @param minutes + * @return all applied updates that happened int he last number of minutes + */ @JsonIgnore - private List getSelected(int minutes){ + private List getSelected(int minutes) { if (appliedSpecList == null) return null; - ArrayList c = new ArrayList(appliedSpecList); + ArrayList c = new ArrayList(appliedSpecList); Timestamp expirationDate = new Timestamp(System.currentTimeMillis() - minutes * 1000 * 60); - for (Iterator it = c.iterator(); it.hasNext(); ) + for (Iterator it = c.iterator(); it.hasNext(); ) if (((AppliedEntity) it.next()).getTimestamp().before(expirationDate)) it.remove(); @@ -1149,63 +1224,205 @@ private Stop getStopForStopId(String feedId, String stopId) { return stop; } - + /** + * + * @return all errors, grouped by their type exposing the number of same occurrences + */ public Map getErrors() { + return errorMap; } + /** + * + * @return the set of all errors occurring in the last block of updates + */ + @JsonIgnore + public Set getLastErrors(){ + return lastErrors; + } + + /** + * + * @return the number of applied updates per tripId + */ public Map getApplied() { + return appliedMap; } + /** + * + * @return return the number of received updates er tripId + */ public Map getReceived() { + return receivedMap; } + /** + * + * @return the number of updates per type + */ public Map getTypes() { + return typeMap; } + /** + * + * @param feedId + * @return the number of applied updates for feedId + */ + @JsonIgnore + public Long getAppliedPerFeed(String feedId) { + Object ret = toMap(appliedSpecList, "feedId", feedId); + if (ret == null) + return null; + return (Long) ret; + } + + /** + * + * @return the number of applied per feedId + */ public Map getAppliedPerFeed() { - return toMap(appliedSpecList, "feedId"); + Object ret = toMap(appliedSpecList, "feedId", null); + if (ret == null) + return null; + return (Map) ret; } + /** + * + * @return the number of applied per tripId + */ public Map getAppliedPerTrip() { - return toMap(appliedSpecList, "tripId"); + Object ret = toMap(appliedSpecList, "tripId", null); + if (ret == null) + return null; + return (Map) ret; } + /** + * + * @return all the number of applied per type + */ public Map getAppliedPerType() { - return toMap(appliedSpecList, "type"); + Object ret = toMap(appliedSpecList, "type", null); + if (ret == null) + return null; + return (Map) ret; } - public Map>> getAppliedTypePerFeedPerTrip() { + /** + * + * @return all applied updates grouped by type, feedId and tripId + */ + public Map>> getAppliedTypePerFeedPerTrip() { return toMap2(appliedSpecList); } - public String getLastUpdate() { - return lastUpdate.toString(); + /** + * + * @param feedId + * @param tripId + * @return all applied updates per feedId and tripId, grouped by type exposing the number + */ + @JsonIgnore + public Map getAppliedTypePerFeedPerTrip(String feedId, String tripId) { + return toMap2(appliedSpecList, feedId, tripId); + + } + + /** + * + * @return timestamp last applied updates + */ + private Long getLastApplied() { + if (lastUpdate == null) + return null; + return lastUpdate.getTime(); } - public String getLastReceived() { - return lastReceived.toString(); + /** + * + * @return timestamp last received updates + */ + private long getLastReceived() { + + if (lastReceived == null) + return 0L; + return lastReceived.getTime(); } - public Map> getAppliedTypePerFeedPerTripNumber() { - return toMap2Number(appliedSpecList); + /** + * + * @return the tripId and timestamp of last applied and received update + */ + public Map getLastAppliedReceived(){ + Map ret = new HashMap<>(); + ret.put("lastUpdatedTrip", lastUpdatedTrip); + ret.put("lastUpdatedTime", getLastApplied()); + ret.put("lastReceivedTrip",lastReceivedTrip); + ret.put("lastReceivedTime",getLastReceived()); + return ret; + } + + /** + * + * @return the number of received and applied updates + */ + public Map getReceivedApplied(){ + Map ret = new HashMap<>(); + ret.put("receivedNum",receivedNum); + ret.put("appliedNum",appliedNum); + return ret; } + + /** + * Classes can addErrors if in relation to TripUpdate + * @param err + */ public void addError(String err) { errorList.add(err); + lastErrors.add(err); } + + /** + * + * @param minutes + * @return information about all updates occurring in the last number of minutes + */ @JsonIgnore public List getAppliedLastMinutes(int minutes) { - return getSelected(minutes); + + Map c = toMapObject(getSelected(minutes)); + + if (c == null) + return null; + List ret = new ArrayList<>(); + int i = 0; + AppliedEntity ap; + for (Map.Entry entry : c.entrySet()) { + ap = (AppliedEntity) entry.getKey(); + ap.setOccurrence(entry.getValue()); + ret.add((AppliedEntity) entry.getKey()); + } + return ret; } + + + } +/** + * Following classes are helpers while creating Statistics for updates + */ class AppliedEntity implements Comparable { @JsonIgnore String type; @@ -1216,6 +1433,16 @@ class AppliedEntity implements Comparable { @JsonIgnore Timestamp timestamp; + public Long getOccurrence() { + return occurrence; + } + + public void setOccurrence(Long occurrence) { + this.occurrence = occurrence; + } + + Long occurrence; + AppliedEntity(String type, String feedId, String tripId) { @@ -1224,6 +1451,7 @@ class AppliedEntity implements Comparable { this.tripId = tripId; this.feedTrip = new FeedTrip(feedId, tripId); this.timestamp = new Timestamp(System.currentTimeMillis()); + this.occurrence = 0L; } public Timestamp getTimestamp() { @@ -1267,16 +1495,33 @@ public int hashCode() { return this.getFeedId().hashCode() * 13 + this.getTripId().hashCode() * 17 + this.getType().hashCode() * 31; } - @Override - public int compareTo(Object T1) { + @Override + public int compareTo(Object o) { + if (o == this) { + return 0; + } + if (o == null) { + return -1; + } + if (!(o instanceof AppliedEntity)) { + return -1; + } + AppliedEntity ft = (AppliedEntity) o; + if (this.getFeedId().compareTo(ft.getFeedId()) == 0) + if (this.getTripId().compareTo(ft.getTripId()) == 0) + if (this.getType().compareTo(ft.getType()) == 0) + return 0; + else + return this.getType().compareTo(ft.getType()); + else return this.getTripId().compareTo(ft.getTripId()); + else return this.getFeedId().compareTo(ft.getFeedId()); - return 1; } - } +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") class FeedTrip { String feedId; String tripId; @@ -1316,8 +1561,4 @@ public String getTripId() { return tripId; } - @Override - public String toString() { - return "feedTrip"; - } } From 8ac40c08db804f54bf015d83b7c87f16d812c5ea Mon Sep 17 00:00:00 2001 From: Milena Mandic Date: Fri, 27 Jan 2017 21:55:32 +0100 Subject: [PATCH 12/12] Web service for status on real time updates v.3 --- .../api/resource/UpdaterStatusResource.java | 85 ++++++++++++------- 1 file changed, 53 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/opentripplanner/api/resource/UpdaterStatusResource.java b/src/main/java/org/opentripplanner/api/resource/UpdaterStatusResource.java index 00dd08a56a3..c43dbd826b6 100644 --- a/src/main/java/org/opentripplanner/api/resource/UpdaterStatusResource.java +++ b/src/main/java/org/opentripplanner/api/resource/UpdaterStatusResource.java @@ -1,9 +1,7 @@ package org.opentripplanner.api.resource; -import com.google.transit.realtime.GtfsRealtime; import org.opentripplanner.standalone.OTPServer; import org.opentripplanner.standalone.Router; -import org.opentripplanner.updater.GraphUpdater; import org.opentripplanner.updater.GraphUpdaterManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,13 +29,22 @@ public class UpdaterStatusResource { Router router; + /** + * + * @param otpServer + * @param routerId id of the router + */ public UpdaterStatusResource(@Context OTPServer otpServer, @PathParam("routerId") String routerId) { router = otpServer.getRouter(routerId); } /** * - * @return most of the important data calculated by TimetableSnapshotSource + * Returns view of the important data on real time status updates. + * + * @return general information about updates(errors of the updates, types + * of updates, received and applied updates) + * */ @GET public Response getDescription() { @@ -50,21 +57,27 @@ public Response getDescription() { /** * - * @return a list of all agencies in the graph for the given feedId. + * Returns all agencies for specific feed. + * @param feedId id of the feed + * @return a list of all agencies in the graph for the specific feed, + * if such exist otherwise return null */ @GET @Path("/agency/{feedId}") public Response getAgencies(@PathParam("feedId") String feedId) { GraphUpdaterManager updaterManager = router.graph.updaterManager; if (updaterManager == null) { - return Response.status(Response.Status.NOT_FOUND).entity("There is no updaters running.").build(); + return Response.status(Response.Status.NOT_FOUND).entity("There are no updaters running.").build(); } return Response.status(Response.Status.OK).entity(updaterManager.getAgency(feedId)).build(); } /** * - * @return status for a specific updater. + * Returns information on specific updater. + * @param updaterId id of the updater + * @return Description for the updater with updater id. + * If such does not exist "Updater does not exist." is reported. */ @GET @Path("/{updaterId}") @@ -78,7 +91,9 @@ public Response getTypeOfUpdater(@PathParam("updaterId") int updaterId) { /** * - * @return correct and all stream addresses + * Returns information on stream addresses. + * @return set of all correct stream addresses and list of information on all + * stream addresses. */ @GET @Path("/stream") @@ -92,7 +107,8 @@ public Response getStreamAddresses() { /** * - * @return types of updater + * Returns types of updaters. + * @return a list of all types of updaters */ @GET @Path("/types") @@ -106,8 +122,10 @@ public Response getTypes() { /** * - * @param updaterId - * @return type of updater for updaterId + * Returns short information on the updater. + * @param updaterId id of the updater + * @return type of updater for updater id, + * if such does not exist "No updater." is reported */ @GET @Path("/types/{id}") @@ -121,8 +139,9 @@ public Response getTypePerId(@PathParam("id") int updaterId) { /** * - * @return all updates grouped by tripid, - * exposing the number of time tripid showed in updates + * Returns a list of received updates. + * @return all updates grouped by trip id, + * exposing the number of times each trip was updated */ @GET @Path("/updates") @@ -136,7 +155,7 @@ public Response getUpdates() { /** * - * @return the number of updates per type + * Returns the number of updates per type. */ @GET @Path("/updates/types") @@ -149,8 +168,8 @@ public Response getUpdatesTypes() { } /** - * - * @return the number of updates applied per tripId + * Returns the number of applied updates. + * @return the number of applied updates grouped by trip id */ @GET @Path("/updates/applied") @@ -163,8 +182,8 @@ public Response getApplied() { } /** - * - * @return the errors for updates grouped by its String representation + * Returns errors that occurred for non-applied updates. + * @return errors for updates grouped by its string representation * exposing the number of occurrences for each error */ @GET @@ -178,8 +197,8 @@ public Response getErrors() { } /** - * - * @return the errors for last block of updates + * Returns errors that occurred for last block of non-applied updates. + * @return errors for the last block of updates */ @GET @Path("/updates/errors/last") @@ -193,7 +212,9 @@ public Response getLastErrors() { /** * - * @return time and tripId for the last received and applied update + * Returns information on the last updated trip. + * @return time when update occurred, id of the trip being updated for + * each last received and last applied update */ @GET @Path("/updates/last") @@ -207,7 +228,8 @@ public Response getLastAppliedReceived() { /** * - * @return the ratio between received and applied update + * Returns the ratio between received and applied updates. + * @return the number of received and applied updates */ @GET @Path("/updates/ratio") @@ -220,10 +242,10 @@ public Response getReceivedApplied() { } /** - * - * @param feedId - * @param tripId - * @return the number of updates for feedId and tripId grouped by type + * Returns the number of applied updates for feed and trip. + * @param feedId id of the feed + * @param tripId id of the trip + * @return the number of updates for specific feed and trip, grouped by type */ @GET @Path("/updates/applied/feed/{feedId}/trip/{tripId}") @@ -236,9 +258,9 @@ public Response getUpdatesPerFeedPerTrip(@PathParam("feedId") String feedId, @Pa } /** - * - * @param feedId - * @return the number of updates applied for feedId + * Returns the number of applied updates for provided feed. + * @param feedId id of the feed + * @return the number of applied updates for specific feed */ @GET @Path("/updates/applied/feed/{feedId}") @@ -251,10 +273,9 @@ public Response getAppliedPerFeed(@PathParam("feedId") String feedId) { } /** - * - * @param minutes - * @return information about updates that occurred - * in the last number of minutes + * Returns the information on applied updates not older than provided parameter. + * @param minutes the number of minutes (all updates older than 60 minutes are discarded) + * @return information about applied updates that occurred in the last number of minutes */ @GET @Path("updates/applied/{lastMinutes}")