From 529bf23370537c61ddc814277e20800634244ba9 Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Tue, 27 Aug 2024 23:07:45 +0200 Subject: [PATCH 01/53] Testing method --- .../exceptions/NoWeatherDataException.java | 25 +++++++++++++++++++ .../io/source/csv/CsvWeatherSource.java | 18 ++++++++++--- 2 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java new file mode 100644 index 000000000..2f6405af3 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java @@ -0,0 +1,25 @@ +/* + * © 2021. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.exceptions; + +/** + * Exception that should be used whenever no weather data is available in a weather data source. + * + * @version 0.1 + * @since 27.08.24 + */ +public class NoWeatherDataException extends SourceException { + + private static final long serialVersionUID = 123456789L; + + public NoWeatherDataException(final String message) { + super(message); + } + + public NoWeatherDataException(final String message, final Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 2b8a368a7..88f26a60f 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -8,6 +8,7 @@ import static edu.ie3.datamodel.utils.validation.UniquenessValidationUtils.checkWeatherUniqueness; import edu.ie3.datamodel.exceptions.DuplicateEntitiesException; +import edu.ie3.datamodel.exceptions.NoWeatherDataException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.connectors.CsvFileConnector; @@ -155,13 +156,22 @@ protected IndividualTimeSeries mergeTimeSeries( // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- private Map> getWeatherTimeSeries() - throws SourceException { - /* Get only weather time series meta information */ + throws SourceException { + // Get only weather time series meta information Collection weatherCsvMetaInformation = - dataSource.getCsvIndividualTimeSeriesMetaInformation(ColumnScheme.WEATHER).values(); - return readWeatherTimeSeries(Set.copyOf(weatherCsvMetaInformation), dataSource.connector); + dataSource.getCsvIndividualTimeSeriesMetaInformation(ColumnScheme.WEATHER).values(); + + Map> weatherTimeSeries = + readWeatherTimeSeries(Set.copyOf(weatherCsvMetaInformation), dataSource.connector); + + if (weatherTimeSeries.isEmpty()) { + throw new NoWeatherDataException("No weather data available from the CSV source."); + } + + return weatherTimeSeries; } + /** * Reads weather data to time series and maps them coordinate wise * From f72bb750c4a69051d43ac080d7cfa45341aeb213 Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Tue, 27 Aug 2024 23:14:43 +0200 Subject: [PATCH 02/53] Fixed format --- .../exceptions/NoWeatherDataException.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java index 2f6405af3..bebe70ccf 100644 --- a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java +++ b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java @@ -12,14 +12,13 @@ * @since 27.08.24 */ public class NoWeatherDataException extends SourceException { + private static final long serialVersionUID = 123456789L; - private static final long serialVersionUID = 123456789L; + public NoWeatherDataException(final String message) { + super(message); + } - public NoWeatherDataException(final String message) { - super(message); - } - - public NoWeatherDataException(final String message, final Throwable cause) { - super(message, cause); - } + public NoWeatherDataException(final String message, final Throwable cause) { + super(message, cause); + } } From 9447e8b92e3932e9fe9dbe58728d19a30ea4e796 Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Tue, 27 Aug 2024 23:20:18 +0200 Subject: [PATCH 03/53] Fixed format --- .../ie3/datamodel/exceptions/NoWeatherDataException.java | 6 +++--- .../edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java index bebe70ccf..d971aba2b 100644 --- a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java +++ b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java @@ -2,7 +2,7 @@ * © 2021. TU Dortmund University, * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation - */ +*/ package edu.ie3.datamodel.exceptions; /** @@ -15,10 +15,10 @@ public class NoWeatherDataException extends SourceException { private static final long serialVersionUID = 123456789L; public NoWeatherDataException(final String message) { - super(message); + super(message); } public NoWeatherDataException(final String message, final Throwable cause) { - super(message, cause); + super(message, cause); } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 88f26a60f..213dd5817 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -156,13 +156,13 @@ protected IndividualTimeSeries mergeTimeSeries( // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- private Map> getWeatherTimeSeries() - throws SourceException { + throws SourceException { // Get only weather time series meta information Collection weatherCsvMetaInformation = - dataSource.getCsvIndividualTimeSeriesMetaInformation(ColumnScheme.WEATHER).values(); + dataSource.getCsvIndividualTimeSeriesMetaInformation(ColumnScheme.WEATHER).values(); Map> weatherTimeSeries = - readWeatherTimeSeries(Set.copyOf(weatherCsvMetaInformation), dataSource.connector); + readWeatherTimeSeries(Set.copyOf(weatherCsvMetaInformation), dataSource.connector); if (weatherTimeSeries.isEmpty()) { throw new NoWeatherDataException("No weather data available from the CSV source."); @@ -171,7 +171,6 @@ private Map> getWeatherTimeSeries() return weatherTimeSeries; } - /** * Reads weather data to time series and maps them coordinate wise * From e0828af9333ac91cbeb5233ddfa52bd97c471bf6 Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Tue, 27 Aug 2024 23:34:23 +0200 Subject: [PATCH 04/53] Fixed Test --- .../exceptions/NoWeatherDataException.java | 5 ++++ .../io/source/csv/CsvWeatherSource.java | 30 +++++++++++-------- .../csv/CsvWeatherSourceCosmoTest.groovy | 1 - 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java index d971aba2b..309f2b711 100644 --- a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java +++ b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java @@ -12,6 +12,7 @@ * @since 27.08.24 */ public class NoWeatherDataException extends SourceException { + private static final long serialVersionUID = 123456789L; public NoWeatherDataException(final String message) { @@ -21,4 +22,8 @@ public NoWeatherDataException(final String message) { public NoWeatherDataException(final String message, final Throwable cause) { super(message, cause); } + + public NoWeatherDataException(String filePath, String reason) { + super("No weather data available from the source. File: " + filePath + ". Reason: " + reason); + } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 213dd5817..146822ef7 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -165,28 +165,23 @@ private Map> getWeatherTimeSeries() readWeatherTimeSeries(Set.copyOf(weatherCsvMetaInformation), dataSource.connector); if (weatherTimeSeries.isEmpty()) { - throw new NoWeatherDataException("No weather data available from the CSV source."); + throw new NoWeatherDataException( + "CSV source", "No weather data was found after processing the CSV files."); } return weatherTimeSeries; } - /** - * Reads weather data to time series and maps them coordinate wise - * - * @param weatherMetaInformation Data needed for reading - * @return time series mapped to the represented coordinate - */ private Map> readWeatherTimeSeries( Set weatherMetaInformation, CsvFileConnector connector) throws SourceException { + final Map> weatherTimeSeries = new HashMap<>(); Function, Optional>> fieldToValueFunction = this::buildWeatherValue; - /* Reading in weather time series */ + for (CsvIndividualTimeSeriesMetaInformation data : weatherMetaInformation) { - // we need a reader for each file try (BufferedReader reader = connector.initReader(data.getFullFilePath())) { buildStreamWithFieldsToAttributesMap(reader) .getOrThrow() @@ -195,9 +190,6 @@ private Map> readWeatherTimeSeries( .collect(Collectors.groupingBy(tbv -> tbv.getValue().getCoordinate())) .forEach( (point, timeBasedValues) -> { - // We have to generate a random UUID as we'd risk running into duplicate key - // issues - // otherwise IndividualTimeSeries timeSeries = new IndividualTimeSeries<>(UUID.randomUUID(), new HashSet<>(timeBasedValues)); if (weatherTimeSeries.containsKey(point)) { @@ -216,9 +208,21 @@ private Map> readWeatherTimeSeries( } catch (ValidationException e) { throw new SourceException("Validation failed for file " + data.getFullFilePath() + ".", e); } + + // Check if no data was read for this file + if (weatherTimeSeries.isEmpty()) { + throw new NoWeatherDataException( + String.valueOf(data.getFullFilePath()), "No weather data found in the file."); + } + } + + // Check if no data was read and throw an exception + if (weatherTimeSeries.isEmpty()) { + throw new NoWeatherDataException( + "CSV source", "No weather data found after processing all files."); } - // checking the uniqueness before returning the time series + // Checking the uniqueness before returning the time series List exceptions = Try.getExceptions( weatherTimeSeries.values().stream() diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy index cc5e709f2..0e5ef6cea 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -80,7 +80,6 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeSeries193187) } - def "A CsvWeatherSource can read all weather data in a given time interval"() { given: def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) From 6c4c0f82ff79e86bdbfcf39595856a366485f05d Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Tue, 27 Aug 2024 23:42:48 +0200 Subject: [PATCH 05/53] Revert "Testing method" This reverts commit 529bf233 --- .../exceptions/NoWeatherDataException.java | 29 ------------- .../io/source/csv/CsvWeatherSource.java | 41 +++++++------------ 2 files changed, 14 insertions(+), 56 deletions(-) delete mode 100644 src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java deleted file mode 100644 index 309f2b711..000000000 --- a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * © 2021. TU Dortmund University, - * Institute of Energy Systems, Energy Efficiency and Energy Economics, - * Research group Distribution grid planning and operation -*/ -package edu.ie3.datamodel.exceptions; - -/** - * Exception that should be used whenever no weather data is available in a weather data source. - * - * @version 0.1 - * @since 27.08.24 - */ -public class NoWeatherDataException extends SourceException { - - private static final long serialVersionUID = 123456789L; - - public NoWeatherDataException(final String message) { - super(message); - } - - public NoWeatherDataException(final String message, final Throwable cause) { - super(message, cause); - } - - public NoWeatherDataException(String filePath, String reason) { - super("No weather data available from the source. File: " + filePath + ". Reason: " + reason); - } -} diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 146822ef7..2b8a368a7 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -8,7 +8,6 @@ import static edu.ie3.datamodel.utils.validation.UniquenessValidationUtils.checkWeatherUniqueness; import edu.ie3.datamodel.exceptions.DuplicateEntitiesException; -import edu.ie3.datamodel.exceptions.NoWeatherDataException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.connectors.CsvFileConnector; @@ -157,31 +156,28 @@ protected IndividualTimeSeries mergeTimeSeries( private Map> getWeatherTimeSeries() throws SourceException { - // Get only weather time series meta information + /* Get only weather time series meta information */ Collection weatherCsvMetaInformation = dataSource.getCsvIndividualTimeSeriesMetaInformation(ColumnScheme.WEATHER).values(); - - Map> weatherTimeSeries = - readWeatherTimeSeries(Set.copyOf(weatherCsvMetaInformation), dataSource.connector); - - if (weatherTimeSeries.isEmpty()) { - throw new NoWeatherDataException( - "CSV source", "No weather data was found after processing the CSV files."); - } - - return weatherTimeSeries; + return readWeatherTimeSeries(Set.copyOf(weatherCsvMetaInformation), dataSource.connector); } + /** + * Reads weather data to time series and maps them coordinate wise + * + * @param weatherMetaInformation Data needed for reading + * @return time series mapped to the represented coordinate + */ private Map> readWeatherTimeSeries( Set weatherMetaInformation, CsvFileConnector connector) throws SourceException { - final Map> weatherTimeSeries = new HashMap<>(); Function, Optional>> fieldToValueFunction = this::buildWeatherValue; - + /* Reading in weather time series */ for (CsvIndividualTimeSeriesMetaInformation data : weatherMetaInformation) { + // we need a reader for each file try (BufferedReader reader = connector.initReader(data.getFullFilePath())) { buildStreamWithFieldsToAttributesMap(reader) .getOrThrow() @@ -190,6 +186,9 @@ private Map> readWeatherTimeSeries( .collect(Collectors.groupingBy(tbv -> tbv.getValue().getCoordinate())) .forEach( (point, timeBasedValues) -> { + // We have to generate a random UUID as we'd risk running into duplicate key + // issues + // otherwise IndividualTimeSeries timeSeries = new IndividualTimeSeries<>(UUID.randomUUID(), new HashSet<>(timeBasedValues)); if (weatherTimeSeries.containsKey(point)) { @@ -208,21 +207,9 @@ private Map> readWeatherTimeSeries( } catch (ValidationException e) { throw new SourceException("Validation failed for file " + data.getFullFilePath() + ".", e); } - - // Check if no data was read for this file - if (weatherTimeSeries.isEmpty()) { - throw new NoWeatherDataException( - String.valueOf(data.getFullFilePath()), "No weather data found in the file."); - } - } - - // Check if no data was read and throw an exception - if (weatherTimeSeries.isEmpty()) { - throw new NoWeatherDataException( - "CSV source", "No weather data found after processing all files."); } - // Checking the uniqueness before returning the time series + // checking the uniqueness before returning the time series List exceptions = Try.getExceptions( weatherTimeSeries.values().stream() From 743e99aacd5cc44e4115ec5f3ce451ecd9d5fd22 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 4 Sep 2024 22:04:53 +0200 Subject: [PATCH 06/53] Testing NoWeatherDataException --- .../exceptions/NoWeatherDataException.java | 20 +++++++++++++++++++ .../datamodel/io/source/WeatherSource.java | 3 ++- .../io/source/csv/CsvWeatherSource.java | 11 +++++++++- 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java new file mode 100644 index 000000000..66b4e9ba4 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java @@ -0,0 +1,20 @@ +/* + * © 2021. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.exceptions; + +/** + * Exception that should be used whenever no weather data is received{@link + * edu.ie3.datamodel.io.source.DataSource} + * + * @version 0.1 + * @since 04.09.24 + */ +public class NoWeatherDataException extends Exception { + + public NoWeatherDataException(final String message) { + super(message); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index a7a056d85..4cc4e89ad 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source; +import edu.ie3.datamodel.exceptions.NoWeatherDataException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; @@ -55,7 +56,7 @@ public void validate() throws ValidationException { // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- public abstract Map> getWeather( - ClosedInterval timeInterval) throws SourceException; + ClosedInterval timeInterval) throws SourceException, NoWeatherDataException; public abstract Map> getWeather( ClosedInterval timeInterval, Collection coordinates) diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 2b8a368a7..73bc0f3e6 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -8,6 +8,7 @@ import static edu.ie3.datamodel.utils.validation.UniquenessValidationUtils.checkWeatherUniqueness; import edu.ie3.datamodel.exceptions.DuplicateEntitiesException; +import edu.ie3.datamodel.exceptions.NoWeatherDataException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.connectors.CsvFileConnector; @@ -90,7 +91,15 @@ public Optional> getSourceFields() { @Override public Map> getWeather( - ClosedInterval timeInterval) { + ClosedInterval timeInterval) throws NoWeatherDataException { + + Map> result = + trimMapToInterval(coordinateToTimeSeries, timeInterval); + + if (result == null || result.isEmpty()) { + throw new NoWeatherDataException("No weather data found."); + } + return trimMapToInterval(coordinateToTimeSeries, timeInterval); } From 74f4816ed2f99340fa57c7f2dd3239869ac6b5bc Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 4 Sep 2024 22:34:43 +0200 Subject: [PATCH 07/53] Handled NoWeatherData in for CSV Source --- .../ie3/datamodel/io/source/WeatherSource.java | 6 +++--- .../io/source/csv/CsvWeatherSource.java | 17 ++++++++++++----- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index 4cc4e89ad..299f95a64 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -60,16 +60,16 @@ public abstract Map> getWeather( public abstract Map> getWeather( ClosedInterval timeInterval, Collection coordinates) - throws SourceException; + throws SourceException, NoWeatherDataException; public abstract Optional> getWeather( - ZonedDateTime date, Point coordinate) throws SourceException; + ZonedDateTime date, Point coordinate) throws SourceException, NoWeatherDataException; public abstract Map> getTimeKeysAfter(ZonedDateTime time) throws SourceException; public List getTimeKeysAfter(ZonedDateTime time, Point coordinate) - throws SourceException { + throws SourceException { return getTimeKeysAfter(time).getOrDefault(coordinate, Collections.emptyList()); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 73bc0f3e6..bbafeba37 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -100,23 +100,30 @@ public Map> getWeather( throw new NoWeatherDataException("No weather data found."); } - return trimMapToInterval(coordinateToTimeSeries, timeInterval); + return result; } @Override public Map> getWeather( - ClosedInterval timeInterval, Collection coordinates) { + ClosedInterval timeInterval, Collection coordinates) throws NoWeatherDataException { Map> filteredMap = coordinateToTimeSeries.entrySet().stream() .filter(entry -> coordinates.contains(entry.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - return trimMapToInterval(filteredMap, timeInterval); + + Map> result = + trimMapToInterval(filteredMap, timeInterval); + + if(result == null || result.isEmpty()) { + throw new NoWeatherDataException("No weather data found."); + } + return result; } @Override - public Optional> getWeather(ZonedDateTime date, Point coordinate) { + public Optional> getWeather(ZonedDateTime date, Point coordinate) throws NoWeatherDataException { IndividualTimeSeries timeSeries = coordinateToTimeSeries.get(coordinate); - if (timeSeries == null) return Optional.empty(); + if (timeSeries == null) throw new NoWeatherDataException("No weather data for given coordinates."); return timeSeries.getTimeBasedValue(date); } From 8e8bed0b61e9f41d71f33b59314f711a9577e283 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 4 Sep 2024 22:35:09 +0200 Subject: [PATCH 08/53] Handled NoWeatherData in for CSV Source --- .../edu/ie3/datamodel/io/source/WeatherSource.java | 2 +- .../datamodel/io/source/csv/CsvWeatherSource.java | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index 299f95a64..abcb1aa72 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -69,7 +69,7 @@ public abstract Map> getTimeKeysAfter(ZonedDateTime t throws SourceException; public List getTimeKeysAfter(ZonedDateTime time, Point coordinate) - throws SourceException { + throws SourceException { return getTimeKeysAfter(time).getOrDefault(coordinate, Collections.emptyList()); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index bbafeba37..a5e10133b 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -105,25 +105,28 @@ public Map> getWeather( @Override public Map> getWeather( - ClosedInterval timeInterval, Collection coordinates) throws NoWeatherDataException { + ClosedInterval timeInterval, Collection coordinates) + throws NoWeatherDataException { Map> filteredMap = coordinateToTimeSeries.entrySet().stream() .filter(entry -> coordinates.contains(entry.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); Map> result = - trimMapToInterval(filteredMap, timeInterval); + trimMapToInterval(filteredMap, timeInterval); - if(result == null || result.isEmpty()) { + if (result == null || result.isEmpty()) { throw new NoWeatherDataException("No weather data found."); } return result; } @Override - public Optional> getWeather(ZonedDateTime date, Point coordinate) throws NoWeatherDataException { + public Optional> getWeather(ZonedDateTime date, Point coordinate) + throws NoWeatherDataException { IndividualTimeSeries timeSeries = coordinateToTimeSeries.get(coordinate); - if (timeSeries == null) throw new NoWeatherDataException("No weather data for given coordinates."); + if (timeSeries == null) + throw new NoWeatherDataException("No weather data for given coordinates."); return timeSeries.getTimeBasedValue(date); } From e0bcfb66c0c4dec7862691fa6041955c7d0a82e3 Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Thu, 12 Sep 2024 00:54:38 +0200 Subject: [PATCH 09/53] Adapted exception throwing for getWeather() which returns optional and changed NoDataException --- ...ataException.java => NoDataException.java} | 6 +++--- .../datamodel/io/source/WeatherSource.java | 8 +++---- .../io/source/csv/CsvWeatherSource.java | 21 +++++++++++-------- 3 files changed, 19 insertions(+), 16 deletions(-) rename src/main/java/edu/ie3/datamodel/exceptions/{NoWeatherDataException.java => NoDataException.java} (71%) diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java similarity index 71% rename from src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java rename to src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java index 66b4e9ba4..e8245730b 100644 --- a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java +++ b/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java @@ -1,5 +1,5 @@ /* - * © 2021. TU Dortmund University, + * © 2024. TU Dortmund University, * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation */ @@ -12,9 +12,9 @@ * @version 0.1 * @since 04.09.24 */ -public class NoWeatherDataException extends Exception { +public class NoDataException extends Exception { - public NoWeatherDataException(final String message) { + public NoDataException(final String message) { super(message); } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index abcb1aa72..c8442c03a 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -5,7 +5,7 @@ */ package edu.ie3.datamodel.io.source; -import edu.ie3.datamodel.exceptions.NoWeatherDataException; +import edu.ie3.datamodel.exceptions.NoDataException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; @@ -56,14 +56,14 @@ public void validate() throws ValidationException { // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- public abstract Map> getWeather( - ClosedInterval timeInterval) throws SourceException, NoWeatherDataException; + ClosedInterval timeInterval) throws SourceException, NoDataException; public abstract Map> getWeather( ClosedInterval timeInterval, Collection coordinates) - throws SourceException, NoWeatherDataException; + throws SourceException, NoDataException; public abstract Optional> getWeather( - ZonedDateTime date, Point coordinate) throws SourceException, NoWeatherDataException; + ZonedDateTime date, Point coordinate) throws SourceException, NoDataException; public abstract Map> getTimeKeysAfter(ZonedDateTime time) throws SourceException; diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index a5e10133b..449b8647d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -8,7 +8,7 @@ import static edu.ie3.datamodel.utils.validation.UniquenessValidationUtils.checkWeatherUniqueness; import edu.ie3.datamodel.exceptions.DuplicateEntitiesException; -import edu.ie3.datamodel.exceptions.NoWeatherDataException; +import edu.ie3.datamodel.exceptions.NoDataException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.connectors.CsvFileConnector; @@ -91,13 +91,13 @@ public Optional> getSourceFields() { @Override public Map> getWeather( - ClosedInterval timeInterval) throws NoWeatherDataException { + ClosedInterval timeInterval) throws NoDataException { Map> result = trimMapToInterval(coordinateToTimeSeries, timeInterval); if (result == null || result.isEmpty()) { - throw new NoWeatherDataException("No weather data found."); + throw new NoDataException("No weather data found."); } return result; @@ -106,7 +106,7 @@ public Map> getWeather( @Override public Map> getWeather( ClosedInterval timeInterval, Collection coordinates) - throws NoWeatherDataException { + throws NoDataException { Map> filteredMap = coordinateToTimeSeries.entrySet().stream() .filter(entry -> coordinates.contains(entry.getKey())) @@ -116,18 +116,21 @@ public Map> getWeather( trimMapToInterval(filteredMap, timeInterval); if (result == null || result.isEmpty()) { - throw new NoWeatherDataException("No weather data found."); + throw new NoDataException("No weather data found."); } return result; } + // TODO + // Remove Optional @Override public Optional> getWeather(ZonedDateTime date, Point coordinate) - throws NoWeatherDataException { + throws NoDataException { IndividualTimeSeries timeSeries = coordinateToTimeSeries.get(coordinate); - if (timeSeries == null) - throw new NoWeatherDataException("No weather data for given coordinates."); - return timeSeries.getTimeBasedValue(date); + return Optional.of( + timeSeries + .getTimeBasedValue(date) + .orElseThrow(() -> new NoDataException("No weather data found"))); } @Override From cee6660f2dd1a2c7ba0d26868770e76e68627088 Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Sat, 14 Sep 2024 16:19:39 +0200 Subject: [PATCH 10/53] Started adding NoDataException for SQL source --- .../ie3/datamodel/io/source/sql/SqlWeatherSource.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 358169694..48a28cc5d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -7,6 +7,7 @@ import static edu.ie3.datamodel.io.source.sql.SqlDataSource.createBaseQueryString; +import edu.ie3.datamodel.exceptions.NoDataException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.SqlConnector; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; @@ -92,7 +93,7 @@ public Optional> getSourceFields() { @Override public Map> getWeather( - ClosedInterval timeInterval) throws SourceException { + ClosedInterval timeInterval) throws NoDataException, SourceException { List> timeBasedValues = buildTimeBasedValues( weatherFactory, @@ -102,6 +103,9 @@ public Map> getWeather( ps.setTimestamp(1, Timestamp.from(timeInterval.getLower().toInstant())); ps.setTimestamp(2, Timestamp.from(timeInterval.getUpper().toInstant())); })); + if(timeBasedValues.isEmpty()) { + throw new NoDataException("No weather data found"); + } return mapWeatherValuesToPoints(timeBasedValues); } @@ -137,7 +141,7 @@ public Map> getWeather( @Override public Optional> getWeather(ZonedDateTime date, Point coordinate) - throws SourceException { + throws SourceException, NoDataException { Optional coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { log.warn("Unable to match coordinate {} to a coordinate ID", coordinate); @@ -154,7 +158,7 @@ public Optional> getWeather(ZonedDateTime date, Poi ps.setTimestamp(2, Timestamp.from(date.toInstant())); })); - if (timeBasedValues.isEmpty()) return Optional.empty(); + if (timeBasedValues.isEmpty()) throw new NoDataException("No weather data found"); if (timeBasedValues.size() > 1) log.warn("Retrieved more than one result value, using the first"); return Optional.of(timeBasedValues.get(0)); From f7a21d46f48edb21b64aa0bb67b3fbbd46ee730b Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Sat, 14 Sep 2024 16:22:17 +0200 Subject: [PATCH 11/53] Spotless Applied --- .../edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 48a28cc5d..16fc51012 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -103,7 +103,7 @@ public Map> getWeather( ps.setTimestamp(1, Timestamp.from(timeInterval.getLower().toInstant())); ps.setTimestamp(2, Timestamp.from(timeInterval.getUpper().toInstant())); })); - if(timeBasedValues.isEmpty()) { + if (timeBasedValues.isEmpty()) { throw new NoDataException("No weather data found"); } return mapWeatherValuesToPoints(timeBasedValues); @@ -141,7 +141,7 @@ public Map> getWeather( @Override public Optional> getWeather(ZonedDateTime date, Point coordinate) - throws SourceException, NoDataException { + throws SourceException, NoDataException { Optional coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { log.warn("Unable to match coordinate {} to a coordinate ID", coordinate); From 171610b418aee5b74c821c6e5f8c91c7fb9802f7 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Mon, 30 Sep 2024 11:51:24 +0200 Subject: [PATCH 12/53] Removed all occurences of Optional return type for getWeather() methods --- .../ie3/datamodel/io/source/WeatherSource.java | 2 +- .../source/couchbase/CouchbaseWeatherSource.java | 15 +++++++++------ .../datamodel/io/source/csv/CsvWeatherSource.java | 14 +++++++------- .../io/source/influxdb/InfluxDbWeatherSource.java | 7 ++++--- .../datamodel/io/source/sql/SqlWeatherSource.java | 6 +++--- .../source/csv/CsvWeatherSourceCosmoTest.groovy | 6 +++--- .../io/source/sql/SqlWeatherSourceCosmoIT.groovy | 4 ++-- 7 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index c8442c03a..983da9081 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -62,7 +62,7 @@ public abstract Map> getWeather( ClosedInterval timeInterval, Collection coordinates) throws SourceException, NoDataException; - public abstract Optional> getWeather( + public abstract TimeBasedValue getWeather( ZonedDateTime date, Point coordinate) throws SourceException, NoDataException; public abstract Map> getTimeKeysAfter(ZonedDateTime time) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 8dc56cafe..8f671e22c 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -10,6 +10,7 @@ import com.couchbase.client.java.json.JsonObject; import com.couchbase.client.java.kv.GetResult; import com.couchbase.client.java.query.QueryResult; +import edu.ie3.datamodel.exceptions.NoDataException; import edu.ie3.datamodel.io.connectors.CouchbaseConnector; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; @@ -143,25 +144,27 @@ public Map> getWeather( } @Override - public Optional> getWeather(ZonedDateTime date, Point coordinate) { + public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws NoDataException { Optional coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { logger.warn("Unable to match coordinate {} to a coordinate ID", coordinate); - return Optional.empty(); + throw new NoDataException("No coordinate ID found for the given point."); } try { CompletableFuture futureResult = connector.get(generateWeatherKey(date, coordinateId.get())); GetResult getResult = futureResult.join(); JsonObject jsonWeatherInput = getResult.contentAsObject(); - return toTimeBasedWeatherValue(jsonWeatherInput); + return toTimeBasedWeatherValue(jsonWeatherInput).orElseThrow(() -> new NoDataException("No valid weather data found for the given date and coordinate.")); } catch (DecodingFailureException ex) { logger.error("Decoding to TimeBasedWeatherValue failed!", ex); - return Optional.empty(); + throw new NoDataException("Failed to decode weather data."); } catch (DocumentNotFoundException ex) { - return Optional.empty(); + throw new NoDataException("Weather document not found."); } catch (CompletionException ex) { - if (ex.getCause() instanceof DocumentNotFoundException) return Optional.empty(); + if (ex.getCause() instanceof DocumentNotFoundException) { + throw new NoDataException("Weather document not found in the completion stage."); + } else throw ex; } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 449b8647d..4bf71fe7a 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -121,16 +121,16 @@ public Map> getWeather( return result; } - // TODO - // Remove Optional @Override - public Optional> getWeather(ZonedDateTime date, Point coordinate) + public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws NoDataException { IndividualTimeSeries timeSeries = coordinateToTimeSeries.get(coordinate); - return Optional.of( - timeSeries - .getTimeBasedValue(date) - .orElseThrow(() -> new NoDataException("No weather data found"))); + + if(timeSeries == null){ + throw new NoDataException("No weather data found for the given coordinate"); + } + + return timeSeries.getTimeBasedValue(date).orElseThrow(() -> new NoDataException("No weather data found for the given coordinate")); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 2a23660f5..2e899e513 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source.influxdb; +import edu.ie3.datamodel.exceptions.NoDataException; import edu.ie3.datamodel.io.connectors.InfluxDbConnector; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; @@ -107,15 +108,15 @@ public Map> getWeather( } @Override - public Optional> getWeather(ZonedDateTime date, Point coordinate) { + public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws NoDataException { Optional coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { - return Optional.empty(); + throw new NoDataException("No coordinate ID found for the given point."); } try (InfluxDB session = connector.getSession()) { String query = createQueryStringForCoordinateAndTime(date, coordinateId.get()); QueryResult queryResult = session.query(new Query(query)); - return filterEmptyOptionals(optTimeBasedValueStream(queryResult)).findFirst(); + return filterEmptyOptionals(optTimeBasedValueStream(queryResult)).findFirst().orElseThrow(() -> new NoDataException("No weather data available for the given date and coordinate.")); } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 16fc51012..2b328e93e 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -140,12 +140,12 @@ public Map> getWeather( } @Override - public Optional> getWeather(ZonedDateTime date, Point coordinate) + public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws SourceException, NoDataException { Optional coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { log.warn("Unable to match coordinate {} to a coordinate ID", coordinate); - return Optional.empty(); + throw new NoDataException("No coordinate ID found for the given point."); } List> timeBasedValues = @@ -161,7 +161,7 @@ public Optional> getWeather(ZonedDateTime date, Poi if (timeBasedValues.isEmpty()) throw new NoDataException("No weather data found"); if (timeBasedValues.size() > 1) log.warn("Retrieved more than one result value, using the first"); - return Optional.of(timeBasedValues.get(0)); + return timeBasedValues.get(0); } @Override diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy index 0e5ef6cea..f6142329b 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -46,11 +46,11 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) when: - def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) + def timeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + timeBasedValue != null + equalsIgnoreUUID(timeBasedValue, expectedTimeBasedValue) } def "A CsvWeatherSource can read multiple time series values for multiple coordinates"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index 1a7e0a0d3..fabc43b36 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -56,8 +56,8 @@ class SqlWeatherSourceCosmoIT extends Specification implements TestContainerHelp def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue ) + optTimeBasedValue != null + equalsIgnoreUUID(optTimeBasedValue, expectedTimeBasedValue ) } def "A SqlWeatherSource returns nothing for an invalid coordinate"() { From 08a2679abd6cead93fd3cf4a5f68580fb89fddf1 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Mon, 30 Sep 2024 11:57:51 +0200 Subject: [PATCH 13/53] spotless --- .../edu/ie3/datamodel/io/source/WeatherSource.java | 4 ++-- .../io/source/couchbase/CouchbaseWeatherSource.java | 12 ++++++++---- .../datamodel/io/source/csv/CsvWeatherSource.java | 6 ++++-- .../io/source/influxdb/InfluxDbWeatherSource.java | 10 ++++++++-- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index 983da9081..75c662b10 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -62,8 +62,8 @@ public abstract Map> getWeather( ClosedInterval timeInterval, Collection coordinates) throws SourceException, NoDataException; - public abstract TimeBasedValue getWeather( - ZonedDateTime date, Point coordinate) throws SourceException, NoDataException; + public abstract TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) + throws SourceException, NoDataException; public abstract Map> getTimeKeysAfter(ZonedDateTime time) throws SourceException; diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 8f671e22c..8730013f5 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -144,7 +144,8 @@ public Map> getWeather( } @Override - public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws NoDataException { + public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) + throws NoDataException { Optional coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { logger.warn("Unable to match coordinate {} to a coordinate ID", coordinate); @@ -155,7 +156,11 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin connector.get(generateWeatherKey(date, coordinateId.get())); GetResult getResult = futureResult.join(); JsonObject jsonWeatherInput = getResult.contentAsObject(); - return toTimeBasedWeatherValue(jsonWeatherInput).orElseThrow(() -> new NoDataException("No valid weather data found for the given date and coordinate.")); + return toTimeBasedWeatherValue(jsonWeatherInput) + .orElseThrow( + () -> + new NoDataException( + "No valid weather data found for the given date and coordinate.")); } catch (DecodingFailureException ex) { logger.error("Decoding to TimeBasedWeatherValue failed!", ex); throw new NoDataException("Failed to decode weather data."); @@ -164,8 +169,7 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin } catch (CompletionException ex) { if (ex.getCause() instanceof DocumentNotFoundException) { throw new NoDataException("Weather document not found in the completion stage."); - } - else throw ex; + } else throw ex; } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 4bf71fe7a..010678b07 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -126,11 +126,13 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin throws NoDataException { IndividualTimeSeries timeSeries = coordinateToTimeSeries.get(coordinate); - if(timeSeries == null){ + if (timeSeries == null) { throw new NoDataException("No weather data found for the given coordinate"); } - return timeSeries.getTimeBasedValue(date).orElseThrow(() -> new NoDataException("No weather data found for the given coordinate")); + return timeSeries + .getTimeBasedValue(date) + .orElseThrow(() -> new NoDataException("No weather data found for the given coordinate")); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 2e899e513..f5b4daaf9 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -108,7 +108,8 @@ public Map> getWeather( } @Override - public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws NoDataException { + public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) + throws NoDataException { Optional coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { throw new NoDataException("No coordinate ID found for the given point."); @@ -116,7 +117,12 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin try (InfluxDB session = connector.getSession()) { String query = createQueryStringForCoordinateAndTime(date, coordinateId.get()); QueryResult queryResult = session.query(new Query(query)); - return filterEmptyOptionals(optTimeBasedValueStream(queryResult)).findFirst().orElseThrow(() -> new NoDataException("No weather data available for the given date and coordinate.")); + return filterEmptyOptionals(optTimeBasedValueStream(queryResult)) + .findFirst() + .orElseThrow( + () -> + new NoDataException( + "No weather data available for the given date and coordinate.")); } } From 2a4d3b772546106c71ef4290e2debce561c2468b Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Mon, 30 Sep 2024 12:18:51 +0200 Subject: [PATCH 14/53] fixed tests --- .../io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy | 4 ++-- .../io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index a032cb5bc..07ffa0f03 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -90,8 +90,8 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements TestContain def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + optTimeBasedValue != null + equalsIgnoreUUID(optTimeBasedValue, expectedTimeBasedValue) } def "A CouchbaseWeatherSource can read multiple time series values for multiple coordinates"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy index d2b212a2a..e3ea248c1 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy @@ -65,8 +65,8 @@ class InfluxDbWeatherSourceCosmoIT extends Specification implements TestContaine def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + optTimeBasedValue != null + equalsIgnoreUUID(optTimeBasedValue, expectedTimeBasedValue) } def "An InfluxDbWeatherSource can read multiple time series values for multiple coordinates"() { From 7d449eef899a5b7abf4f1dded7013c92dae1134b Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Mon, 30 Sep 2024 12:33:14 +0200 Subject: [PATCH 15/53] Adapted SourceIconIT tests --- .../io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy | 4 ++-- .../datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy | 4 ++-- .../io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy | 4 ++-- .../ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy index bf3f272f0..d83d37a2e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy @@ -90,8 +90,8 @@ class CouchbaseWeatherSourceIconIT extends Specification implements TestContaine def optTimeBasedValue = source.getWeather(IconWeatherTestData.TIME_15H, IconWeatherTestData.COORDINATE_67775) then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + optTimeBasedValue != null + equalsIgnoreUUID(optTimeBasedValue, expectedTimeBasedValue) } def "A CouchbaseWeatherSource can read multiple time series values for multiple coordinates"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy index 7d90727b2..13bd8914b 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy @@ -43,8 +43,8 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, def optTimeBasedValue = source.getWeather(IconWeatherTestData.TIME_15H, IconWeatherTestData.COORDINATE_67775) then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + optTimeBasedValue != null + equalsIgnoreUUID(optTimeBasedValue, expectedTimeBasedValue) } def "A CsvWeatherSource can read multiple time series values for multiple coordinates"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy index ce3a6dfcb..66c650429 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy @@ -63,8 +63,8 @@ class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSource def optTimeBasedValue = source.getWeather(IconWeatherTestData.TIME_15H , IconWeatherTestData.COORDINATE_67775) then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + optTimeBasedValue != null + equalsIgnoreUUID(optTimeBasedValue, expectedTimeBasedValue) } def "An InfluxDbWeatherSource can read multiple time series values for multiple coordinates"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy index 65cdb72d9..d8194d1cb 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy @@ -54,8 +54,8 @@ class SqlWeatherSourceIconIT extends Specification implements TestContainerHelpe when: def optTimeBasedValue = source.getWeather(IconWeatherTestData.TIME_15H, IconWeatherTestData.COORDINATE_67775) then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue ) + optTimeBasedValue != null + equalsIgnoreUUID(optTimeBasedValue, expectedTimeBasedValue ) } def "A NativeSqlWeatherSource can read multiple timeseries values for multiple coordinates"() { From c83ca936ba1572005c9dafeffcf8a52ae2c5362a Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Mon, 30 Sep 2024 13:22:48 +0200 Subject: [PATCH 16/53] Fixed tests --- .../datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy | 4 ++-- .../io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy | 2 +- .../io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy | 2 +- .../datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy index f6142329b..f24ae8f72 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -46,10 +46,10 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) when: - def timeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: - timeBasedValue != null + optTimeBasedValue != null equalsIgnoreUUID(timeBasedValue, expectedTimeBasedValue) } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy index e3ea248c1..02506f4b8 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy @@ -148,7 +148,7 @@ class InfluxDbWeatherSourceCosmoIT extends Specification implements TestContaine ]) then: - coordinateAtDate == Optional.empty() + coordinateAtDate == null equalsIgnoreUUID(coordinateInInterval, emptyTimeSeries) coordinatesToTimeSeries.keySet() == [validCoordinate].toSet() equalsIgnoreUUID(coordinatesToTimeSeries.get(validCoordinate), timeseries_193186) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy index 66c650429..402ab0116 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy @@ -141,7 +141,7 @@ class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSource ]) then: - coordinateAtDate == Optional.empty() + coordinateAtDate == null equalsIgnoreUUID(coordinateInInterval, emptyTimeSeries) coordinatesToTimeSeries.keySet() == [validCoordinate].toSet() equalsIgnoreUUID(coordinatesToTimeSeries.get(validCoordinate), timeseries67775) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index fabc43b36..dd7425b44 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -65,7 +65,7 @@ class SqlWeatherSourceCosmoIT extends Specification implements TestContainerHelp def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, GeoUtils.buildPoint(89d, 88d)) then: - optTimeBasedValue.empty + optTimeBasedValue == null } def "A SqlWeatherSource can read multiple time series values for multiple coordinates"() { From 34d78aae18ca4a56e225647a238185154d39fbd5 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Mon, 30 Sep 2024 13:49:03 +0200 Subject: [PATCH 17/53] Fixed tests --- .../io/source/influxdb/InfluxDbWeatherSource.java | 13 ++++++++++--- .../datamodel/io/source/sql/SqlWeatherSource.java | 14 ++++++++++---- .../io/source/csv/CsvWeatherSourceCosmoTest.groovy | 2 +- .../influxdb/InfluxDbWeatherSourceCosmoIT.groovy | 2 +- .../influxdb/InfluxDbWeatherSourceIconIT.groovy | 2 +- .../io/source/sql/SqlWeatherSourceCosmoIT.groovy | 2 +- 6 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index f5b4daaf9..9eccf0231 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -110,10 +110,17 @@ public Map> getWeather( @Override public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws NoDataException { - Optional coordinateId = idCoordinateSource.getId(coordinate); - if (coordinateId.isEmpty()) { - throw new NoDataException("No coordinate ID found for the given point."); + Optional coordinateId; + try{ + coordinateId = idCoordinateSource.getId(coordinate); + if (coordinateId.isEmpty()) { + throw new NoDataException("No coordinate ID found for the given point."); + } + }catch (NoDataException e) { + log.error("No data available for coordinate {} and date {}", coordinate, date, e); + return null; } + try (InfluxDB session = connector.getSession()) { String query = createQueryStringForCoordinateAndTime(date, coordinateId.get()); QueryResult queryResult = session.query(new Query(query)); diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 2b328e93e..587c63011 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -142,10 +142,16 @@ public Map> getWeather( @Override public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws SourceException, NoDataException { - Optional coordinateId = idCoordinateSource.getId(coordinate); - if (coordinateId.isEmpty()) { - log.warn("Unable to match coordinate {} to a coordinate ID", coordinate); - throw new NoDataException("No coordinate ID found for the given point."); + Optional coordinateId; + try{ + coordinateId = idCoordinateSource.getId(coordinate); + if (coordinateId.isEmpty()) { + log.warn("Unable to match coordinate {} to a coordinate ID", coordinate); + throw new NoDataException("No coordinate ID found for the given point."); + } + }catch (NoDataException e) { + log.error("No data available for coordinate {} and date {}", coordinate, date, e); + return null; } List> timeBasedValues = diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy index f24ae8f72..07cf7461d 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -50,7 +50,7 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta then: optTimeBasedValue != null - equalsIgnoreUUID(timeBasedValue, expectedTimeBasedValue) + equalsIgnoreUUID(optTimeBasedValue, expectedTimeBasedValue) } def "A CsvWeatherSource can read multiple time series values for multiple coordinates"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy index 02506f4b8..594bf1d6e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy @@ -148,7 +148,7 @@ class InfluxDbWeatherSourceCosmoIT extends Specification implements TestContaine ]) then: - coordinateAtDate == null + assert coordinateAtDate == null equalsIgnoreUUID(coordinateInInterval, emptyTimeSeries) coordinatesToTimeSeries.keySet() == [validCoordinate].toSet() equalsIgnoreUUID(coordinatesToTimeSeries.get(validCoordinate), timeseries_193186) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy index 402ab0116..e4d62131e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy @@ -141,7 +141,7 @@ class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSource ]) then: - coordinateAtDate == null + assert coordinateAtDate == null equalsIgnoreUUID(coordinateInInterval, emptyTimeSeries) coordinatesToTimeSeries.keySet() == [validCoordinate].toSet() equalsIgnoreUUID(coordinatesToTimeSeries.get(validCoordinate), timeseries67775) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index dd7425b44..190890a4f 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -65,7 +65,7 @@ class SqlWeatherSourceCosmoIT extends Specification implements TestContainerHelp def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, GeoUtils.buildPoint(89d, 88d)) then: - optTimeBasedValue == null + assert optTimeBasedValue == null } def "A SqlWeatherSource can read multiple time series values for multiple coordinates"() { From 19d5c3bc75fea5d4d3e57eae7ac66e3bcaf376af Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Mon, 30 Sep 2024 13:51:10 +0200 Subject: [PATCH 18/53] Spotless Apply --- .../datamodel/io/source/influxdb/InfluxDbWeatherSource.java | 4 ++-- .../edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 9eccf0231..5a6e15a70 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -111,12 +111,12 @@ public Map> getWeather( public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws NoDataException { Optional coordinateId; - try{ + try { coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { throw new NoDataException("No coordinate ID found for the given point."); } - }catch (NoDataException e) { + } catch (NoDataException e) { log.error("No data available for coordinate {} and date {}", coordinate, date, e); return null; } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 587c63011..e8bebea94 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -143,13 +143,13 @@ public Map> getWeather( public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws SourceException, NoDataException { Optional coordinateId; - try{ + try { coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { log.warn("Unable to match coordinate {} to a coordinate ID", coordinate); throw new NoDataException("No coordinate ID found for the given point."); } - }catch (NoDataException e) { + } catch (NoDataException e) { log.error("No data available for coordinate {} and date {}", coordinate, date, e); return null; } From ad8946fabba4142f9db3ce420a034a2f6d685588 Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Tue, 1 Oct 2024 00:33:23 +0200 Subject: [PATCH 19/53] Final --- AUTHORS | 3 ++- CHANGELOG.md | 1 + .../couchbase/CouchbaseWeatherSource.java | 10 +++++--- .../influxdb/InfluxDbWeatherSource.java | 22 +++++++++++++++--- .../io/source/sql/SqlWeatherSource.java | 23 +++++++++++-------- .../source/sql/SqlWeatherSourceCosmoIT.groovy | 2 +- 6 files changed, 44 insertions(+), 17 deletions(-) diff --git a/AUTHORS b/AUTHORS index 4c0b256d1..c358b1c4b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -28,4 +28,5 @@ Main Contributers: - Johannes Bao - https://github.com/jo-bao - Julian Hohmann - https://github.com/julianhohmann - Simon Huette - https://github.com/SimonHuette - - Pierre Petersmeier - http://github.com/pierrepetersmeier + - Pierre Petersmeier - https://github.com/pierrepetersmeier + - Philipp Schmelter - https://github.com/PhilippSchmelter diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a2de9e5c..b2e0c8bb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Enhance `TimeSeriesSource` with method to retrieve all time keys after a given key [#543](https://github.com/ie3-institute/PowerSystemDataModel/issues/543) - Enhance `WeatherSource` with method to retrieve all time keys after a given key [#572](https://github.com/ie3-institute/PowerSystemDataModel/issues/572) +- Added explicit handling for cases where no weather data is received from any source [#554](https://github.com/ie3-institute/PowerSystemDataModel/issues/554) ### Fixed diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 8730013f5..3d2b3fbe7 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -105,7 +105,7 @@ public Optional> getSourceFields() { @Override public Map> getWeather( - ClosedInterval timeInterval) { + ClosedInterval timeInterval) throws NoDataException { logger.warn( "By not providing coordinates you are forcing couchbase to check all possible coordinates one by one." + " This is not very performant. Please consider providing specific coordinates instead."); @@ -114,7 +114,8 @@ public Map> getWeather( @Override public Map> getWeather( - ClosedInterval timeInterval, Collection coordinates) { + ClosedInterval timeInterval, Collection coordinates) + throws NoDataException { HashMap> coordinateToTimeSeries = new HashMap<>(); for (Point coordinate : coordinates) { Optional coordinateId = idCoordinateSource.getId(coordinate); @@ -138,7 +139,10 @@ public Map> getWeather( new IndividualTimeSeries<>(weatherInputs); coordinateToTimeSeries.put(coordinate, weatherTimeSeries); } - } else logger.warn("Unable to match coordinate {} to a coordinate ID", coordinate); + } else { + logger.warn("Unable to match coordinate {} to a coordinate ID", coordinate); + throw new NoDataException("No data found"); + } } return coordinateToTimeSeries; } diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 5a6e15a70..6513f7209 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -68,6 +68,9 @@ public Map> getWeather( optTimeBasedValueStream(queryResult); Set> timeBasedValues = filterEmptyOptionals(optValues).collect(Collectors.toSet()); + if (timeBasedValues.isEmpty()) { + throw new NoDataException("No weather data found"); + } Map>> coordinateToValues = timeBasedValues.stream() .collect( @@ -77,6 +80,9 @@ public Map> getWeather( return coordinateToValues.entrySet().stream() .collect( Collectors.toMap(Map.Entry::getKey, e -> new IndividualTimeSeries<>(e.getValue()))); + } catch (NoDataException e) { + log.error("No data available for coordinate", e); + return null; } } @@ -98,11 +104,16 @@ public Map> getWeather( optTimeBasedValueStream(queryResult); Set> timeBasedValues = filterEmptyOptionals(optValues).collect(Collectors.toSet()); + if (timeBasedValues.isEmpty()) { + throw new NoDataException("No weather data found"); + } IndividualTimeSeries timeSeries = new IndividualTimeSeries<>(timeBasedValues); coordinateToTimeSeries.put(entry.getKey(), timeSeries); } } + } catch (NoDataException e) { + return null; } return coordinateToTimeSeries; } @@ -181,9 +192,14 @@ public List getTimeKeysAfter(ZonedDateTime time, Point coordinate * @return weather data for the specified time and coordinate */ public IndividualTimeSeries getWeather( - ClosedInterval timeInterval, Point coordinate) { - Optional coordinateId = idCoordinateSource.getId(coordinate); - if (coordinateId.isEmpty()) { + ClosedInterval timeInterval, Point coordinate) throws NoDataException { + Optional coordinateId; + try { + coordinateId = idCoordinateSource.getId(coordinate); + if (coordinateId.isEmpty()) { + throw new NoDataException("No data for given coordinates"); + } + } catch (NoDataException e) { return new IndividualTimeSeries<>(UUID.randomUUID(), Collections.emptySet()); } try (InfluxDB session = connector.getSession()) { diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index e8bebea94..bd82ee1cc 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -112,15 +112,20 @@ public Map> getWeather( @Override public Map> getWeather( ClosedInterval timeInterval, Collection coordinates) - throws SourceException { - Set coordinateIds = - coordinates.stream() - .map(idCoordinateSource::getId) - .flatMap(Optional::stream) - .collect(Collectors.toSet()); - if (coordinateIds.isEmpty()) { - log.warn("Unable to match coordinates to coordinate ID"); - return Collections.emptyMap(); + throws SourceException, NoDataException { + Set coordinateIds; + try { + coordinateIds = + coordinates.stream() + .map(idCoordinateSource::getId) + .flatMap(Optional::stream) + .collect(Collectors.toSet()); + if (coordinateIds.isEmpty()) { + log.warn("Unable to match coordinates to coordinate ID"); + throw new NoDataException("No coordinates found"); + } + } catch (NoDataException e) { + return null; } List> timeBasedValues = diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index 190890a4f..ef02d2a67 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -107,7 +107,7 @@ class SqlWeatherSourceCosmoIT extends Specification implements TestContainerHelp Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) then: - coordinateToTimeSeries.keySet().empty + coordinateToTimeSeries == null } def "A SqlWeatherSource can read all weather data in a given time interval"() { From 23b732f117178f9b1d3af5bd95fa9b0eedda21f8 Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Tue, 5 Nov 2024 16:47:42 +0100 Subject: [PATCH 20/53] Resolved Conversations --- .../java/edu/ie3/datamodel/exceptions/NoDataException.java | 2 +- .../datamodel/io/source/couchbase/CouchbaseWeatherSource.java | 4 ++-- .../datamodel/io/source/influxdb/InfluxDbWeatherSource.java | 4 ++-- .../edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java index e8245730b..dc52fcb6a 100644 --- a/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java +++ b/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java @@ -6,7 +6,7 @@ package edu.ie3.datamodel.exceptions; /** - * Exception that should be used whenever no weather data is received{@link + * Exception that should be used whenever no data is received{@link * edu.ie3.datamodel.io.source.DataSource} * * @version 0.1 diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 3d2b3fbe7..146270573 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -140,7 +140,7 @@ public Map> getWeather( coordinateToTimeSeries.put(coordinate, weatherTimeSeries); } } else { - logger.warn("Unable to match coordinate {} to a coordinate ID", coordinate); + logger.error("Unable to match coordinate {} to a coordinate ID", coordinate); throw new NoDataException("No data found"); } } @@ -152,7 +152,7 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin throws NoDataException { Optional coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { - logger.warn("Unable to match coordinate {} to a coordinate ID", coordinate); + logger.error("Unable to match coordinate {} to a coordinate ID", coordinate); throw new NoDataException("No coordinate ID found for the given point."); } try { diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 6513f7209..1e70013fc 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -82,7 +82,7 @@ public Map> getWeather( Collectors.toMap(Map.Entry::getKey, e -> new IndividualTimeSeries<>(e.getValue()))); } catch (NoDataException e) { log.error("No data available for coordinate", e); - return null; + return Collections.emptyMap(); } } @@ -113,7 +113,7 @@ public Map> getWeather( } } } catch (NoDataException e) { - return null; + return Collections.emptyMap(); } return coordinateToTimeSeries; } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index bd82ee1cc..e62538a08 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -125,7 +125,7 @@ public Map> getWeather( throw new NoDataException("No coordinates found"); } } catch (NoDataException e) { - return null; + return Collections.emptyMap(); } List> timeBasedValues = From 88e1ec7f58e0c6fdd151649ab622c01bd9221ceb Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Tue, 5 Nov 2024 17:10:24 +0100 Subject: [PATCH 21/53] Fixed Tests --- .../ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index ef02d2a67..a903a2299 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -107,7 +107,7 @@ class SqlWeatherSourceCosmoIT extends Specification implements TestContainerHelp Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) then: - coordinateToTimeSeries == null + coordinateToTimeSeries == Collections.emptyMap() } def "A SqlWeatherSource can read all weather data in a given time interval"() { From 8c2d1f7ad7a47f59be6978364b6def5bc314b7b8 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 20 Aug 2025 23:02:54 +0200 Subject: [PATCH 22/53] Resolved Conversations --- CHANGELOG.md | 1 - .../io/source/couchbase/CouchbaseWeatherSource.java | 6 +++--- .../edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java | 4 ++-- .../datamodel/io/source/influxdb/InfluxDbWeatherSource.java | 6 +++--- .../edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff09e4958..f406c512b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,7 +89,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added load profiles sources [#1106](https://github.com/ie3-institute/PowerSystemDataModel/issues/1106) - Add `v2gSupport` parameter to documentation of `EvcsModel` [#1278](https://github.com/ie3-institute/PowerSystemDataModel/issues/1278) - ### Fixed - Removing opened `SwitchInput` during connectivity check [#1221](https://github.com/ie3-institute/PowerSystemDataModel/issues/1221) - Fixed example in ReadTheDocs [#1244](https://github.com/ie3-institute/PowerSystemDataModel/issues/1244) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index e154aa8fe..0a7e32086 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -142,7 +142,7 @@ public Map> getWeather( } } else { logger.error("Unable to match coordinate {} to a coordinate ID", coordinate); - throw new NoDataException("No data found"); + throw new NoDataException("Unable to match coordinate " + coordinate + " to a coordinate ID"); } } return coordinateToTimeSeries; @@ -154,7 +154,7 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin Optional coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { logger.error("Unable to match coordinate {} to a coordinate ID", coordinate); - throw new NoDataException("No coordinate ID found for the given point."); + throw new NoDataException("No coordinate ID found for the given point: " + coordinate); } try { CompletableFuture futureResult = @@ -168,7 +168,7 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin "No valid weather data found for the given date and coordinate.")); } catch (DecodingFailureException ex) { logger.error("Decoding to TimeBasedWeatherValue failed!", ex); - throw new NoDataException("Failed to decode weather data."); + throw new NoDataException("Failed to decode weather data: " + ex.getMessage()); } catch (DocumentNotFoundException ex) { throw new NoDataException("Weather document not found."); } catch (CompletionException ex) { diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 23958df08..2640f960f 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -127,12 +127,12 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin IndividualTimeSeries timeSeries = coordinateToTimeSeries.get(coordinate); if (timeSeries == null) { - throw new NoDataException("No weather data found for the given coordinate"); + throw new NoDataException("No weather data found for the given coordinate: " + coordinate); } return timeSeries .getTimeBasedValue(date) - .orElseThrow(() -> new NoDataException("No weather data found for the given coordinate")); + .orElseThrow(() -> new NoDataException("No weather data found for the given coordinate: " + coordinate)); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 1e70013fc..77098b8a6 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -125,7 +125,7 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin try { coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { - throw new NoDataException("No coordinate ID found for the given point."); + throw new NoDataException("No coordinate ID found for the given point: " + coordinate); } } catch (NoDataException e) { log.error("No data available for coordinate {} and date {}", coordinate, date, e); @@ -140,7 +140,7 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin .orElseThrow( () -> new NoDataException( - "No weather data available for the given date and coordinate.")); + "No weather data available for the given date " + date + " and coordinate " + coordinate)); } } @@ -197,7 +197,7 @@ public IndividualTimeSeries getWeather( try { coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { - throw new NoDataException("No data for given coordinates"); + throw new NoDataException("No data for given coordinates: " + coordinate); } } catch (NoDataException e) { return new IndividualTimeSeries<>(UUID.randomUUID(), Collections.emptySet()); diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index e62538a08..a18e1bee2 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -152,7 +152,7 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { log.warn("Unable to match coordinate {} to a coordinate ID", coordinate); - throw new NoDataException("No coordinate ID found for the given point."); + throw new NoDataException("No coordinate ID found for the given point: " + coordinate); } } catch (NoDataException e) { log.error("No data available for coordinate {} and date {}", coordinate, date, e); From 5dc0f4aa8b7bffbe5e4b08c82fea92819e660531 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 20 Aug 2025 23:07:00 +0200 Subject: [PATCH 23/53] spotless --- .../io/source/couchbase/CouchbaseWeatherSource.java | 3 ++- .../edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java | 5 ++++- .../datamodel/io/source/influxdb/InfluxDbWeatherSource.java | 5 ++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 0a7e32086..caf19023f 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -142,7 +142,8 @@ public Map> getWeather( } } else { logger.error("Unable to match coordinate {} to a coordinate ID", coordinate); - throw new NoDataException("Unable to match coordinate " + coordinate + " to a coordinate ID"); + throw new NoDataException( + "Unable to match coordinate " + coordinate + " to a coordinate ID"); } } return coordinateToTimeSeries; diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 2640f960f..15d362ac6 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -132,7 +132,10 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin return timeSeries .getTimeBasedValue(date) - .orElseThrow(() -> new NoDataException("No weather data found for the given coordinate: " + coordinate)); + .orElseThrow( + () -> + new NoDataException( + "No weather data found for the given coordinate: " + coordinate)); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 77098b8a6..62a2de11d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -140,7 +140,10 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin .orElseThrow( () -> new NoDataException( - "No weather data available for the given date " + date + " and coordinate " + coordinate)); + "No weather data available for the given date " + + date + + " and coordinate " + + coordinate)); } } From 68ea181bef3ef2421fcc8a8a3a3533f580c90ddf Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 20 Aug 2025 23:33:01 +0200 Subject: [PATCH 24/53] improvements --- .../io/source/csv/CsvWeatherSource.java | 4 +-- .../influxdb/InfluxDbWeatherSource.java | 6 ++--- .../io/source/sql/SqlWeatherSource.java | 6 ++--- .../csv/CsvWeatherSourceIconTest.groovy | 27 +++++++++++++++++++ 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 15d362ac6..a69b9e2a1 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -97,7 +97,7 @@ public Map> getWeather( trimMapToInterval(coordinateToTimeSeries, timeInterval); if (result == null || result.isEmpty()) { - throw new NoDataException("No weather data found."); + throw new NoDataException("No weather data found for the given time interval: " + timeInterval); } return result; @@ -116,7 +116,7 @@ public Map> getWeather( trimMapToInterval(filteredMap, timeInterval); if (result == null || result.isEmpty()) { - throw new NoDataException("No weather data found."); + throw new NoDataException("No weather data found for the given time interval: " + timeInterval); } return result; } diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 62a2de11d..ff1e0774f 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -69,7 +69,7 @@ public Map> getWeather( Set> timeBasedValues = filterEmptyOptionals(optValues).collect(Collectors.toSet()); if (timeBasedValues.isEmpty()) { - throw new NoDataException("No weather data found"); + throw new NoDataException("No weather data found for the given time interval: " + timeInterval); } Map>> coordinateToValues = timeBasedValues.stream() @@ -105,7 +105,7 @@ public Map> getWeather( Set> timeBasedValues = filterEmptyOptionals(optValues).collect(Collectors.toSet()); if (timeBasedValues.isEmpty()) { - throw new NoDataException("No weather data found"); + throw new NoDataException("No weather data found for the given time interval and coordinate: " + entry.getKey()); } IndividualTimeSeries timeSeries = new IndividualTimeSeries<>(timeBasedValues); @@ -129,7 +129,7 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin } } catch (NoDataException e) { log.error("No data available for coordinate {} and date {}", coordinate, date, e); - return null; + throw e; } try (InfluxDB session = connector.getSession()) { diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index a18e1bee2..f333f7dd3 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -104,7 +104,7 @@ public Map> getWeather( ps.setTimestamp(2, Timestamp.from(timeInterval.getUpper().toInstant())); })); if (timeBasedValues.isEmpty()) { - throw new NoDataException("No weather data found"); + throw new NoDataException("No weather data found for the given time interval: " + timeInterval); } return mapWeatherValuesToPoints(timeBasedValues); } @@ -156,7 +156,7 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin } } catch (NoDataException e) { log.error("No data available for coordinate {} and date {}", coordinate, date, e); - return null; + throw e; } List> timeBasedValues = @@ -169,7 +169,7 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin ps.setTimestamp(2, Timestamp.from(date.toInstant())); })); - if (timeBasedValues.isEmpty()) throw new NoDataException("No weather data found"); + if (timeBasedValues.isEmpty()) throw new NoDataException("No weather data found for the given date " + date + " and coordinate " + coordinate); if (timeBasedValues.size() > 1) log.warn("Retrieved more than one result value, using the first"); return timeBasedValues.get(0); diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy index 13bd8914b..18677c887 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source.csv +import edu.ie3.datamodel.exceptions.NoDataException import edu.ie3.datamodel.io.factory.timeseries.IconTimeBasedWeatherValueFactory import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.source.IdCoordinateSource @@ -295,4 +296,30 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, ] actual.get(IconWeatherTestData.COORDINATE_67776) == [IconWeatherTestData.TIME_16H] } + + def "A CsvWeatherSource throws NoDataException when no weather data is found for coordinate"() { + given: + def unknownCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(0.0, 0.0)) + + when: + source.getWeather(IconWeatherTestData.TIME_15H, unknownCoordinate) + + then: + def ex = thrown(NoDataException) + ex.message.contains("No weather data found for the given coordinate") + ex.message.contains(unknownCoordinate.toString()) + } + + def "A CsvWeatherSource throws NoDataException when no weather data is found for coordinate at specific time"() { + given: + def futureTime = IconWeatherTestData.TIME_17H.plusHours(10) + + when: + source.getWeather(futureTime, IconWeatherTestData.COORDINATE_67775) + + then: + def ex = thrown(NoDataException) + ex.message.contains("No weather data found for the given coordinate") + ex.message.contains(IconWeatherTestData.COORDINATE_67775.toString()) + } } From c2117e236be62d5c2b1c84bc780e854131091b33 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 20 Aug 2025 23:35:29 +0200 Subject: [PATCH 25/53] spotless --- .../edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java | 6 ++++-- .../io/source/influxdb/InfluxDbWeatherSource.java | 7 +++++-- .../edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java | 7 +++++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index a69b9e2a1..d0aef8a06 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -97,7 +97,8 @@ public Map> getWeather( trimMapToInterval(coordinateToTimeSeries, timeInterval); if (result == null || result.isEmpty()) { - throw new NoDataException("No weather data found for the given time interval: " + timeInterval); + throw new NoDataException( + "No weather data found for the given time interval: " + timeInterval); } return result; @@ -116,7 +117,8 @@ public Map> getWeather( trimMapToInterval(filteredMap, timeInterval); if (result == null || result.isEmpty()) { - throw new NoDataException("No weather data found for the given time interval: " + timeInterval); + throw new NoDataException( + "No weather data found for the given time interval: " + timeInterval); } return result; } diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index ff1e0774f..9b2f11171 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -69,7 +69,8 @@ public Map> getWeather( Set> timeBasedValues = filterEmptyOptionals(optValues).collect(Collectors.toSet()); if (timeBasedValues.isEmpty()) { - throw new NoDataException("No weather data found for the given time interval: " + timeInterval); + throw new NoDataException( + "No weather data found for the given time interval: " + timeInterval); } Map>> coordinateToValues = timeBasedValues.stream() @@ -105,7 +106,9 @@ public Map> getWeather( Set> timeBasedValues = filterEmptyOptionals(optValues).collect(Collectors.toSet()); if (timeBasedValues.isEmpty()) { - throw new NoDataException("No weather data found for the given time interval and coordinate: " + entry.getKey()); + throw new NoDataException( + "No weather data found for the given time interval and coordinate: " + + entry.getKey()); } IndividualTimeSeries timeSeries = new IndividualTimeSeries<>(timeBasedValues); diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index f333f7dd3..8955864f9 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -104,7 +104,8 @@ public Map> getWeather( ps.setTimestamp(2, Timestamp.from(timeInterval.getUpper().toInstant())); })); if (timeBasedValues.isEmpty()) { - throw new NoDataException("No weather data found for the given time interval: " + timeInterval); + throw new NoDataException( + "No weather data found for the given time interval: " + timeInterval); } return mapWeatherValuesToPoints(timeBasedValues); } @@ -169,7 +170,9 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin ps.setTimestamp(2, Timestamp.from(date.toInstant())); })); - if (timeBasedValues.isEmpty()) throw new NoDataException("No weather data found for the given date " + date + " and coordinate " + coordinate); + if (timeBasedValues.isEmpty()) + throw new NoDataException( + "No weather data found for the given date " + date + " and coordinate " + coordinate); if (timeBasedValues.size() > 1) log.warn("Retrieved more than one result value, using the first"); return timeBasedValues.get(0); From b8ff373245142017667e556ca25cf5e1c76b52d1 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 20 Aug 2025 23:45:04 +0200 Subject: [PATCH 26/53] fixed tests --- .../InfluxDbWeatherSourceCosmoIT.groovy | 28 +++++++++++++------ .../InfluxDbWeatherSourceIconIT.groovy | 28 +++++++++++++------ .../source/sql/SqlWeatherSourceCosmoIT.groovy | 12 ++++++-- 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy index 594bf1d6e..7da8b157b 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source.influxdb +import edu.ie3.datamodel.exceptions.NoDataException import edu.ie3.datamodel.io.connectors.InfluxDbConnector import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries @@ -125,13 +126,12 @@ class InfluxDbWeatherSourceCosmoIT extends Specification implements TestContaine equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).entries, timeseries_193188.entries) } - def "An InfluxDbWeatherSource will return an equivalent to 'empty' when being unable to map a coordinate to its ID"() { + def "An InfluxDbWeatherSource will throw NoDataException when being unable to map a coordinate to its ID"() { given: def validCoordinate = CosmoWeatherTestData.COORDINATE_193186 def invalidCoordinate = GeoUtils.buildPoint(7d, 48d) def time = CosmoWeatherTestData.TIME_15H def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) - def emptyTimeSeries = new IndividualTimeSeries(UUID.randomUUID(), Collections.emptySet()) def timeseries_193186 = new IndividualTimeSeries(null, [ new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), @@ -139,17 +139,29 @@ class InfluxDbWeatherSourceCosmoIT extends Specification implements TestContaine new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H) ] as Set) - when: - def coordinateAtDate = source.getWeather(time, invalidCoordinate) - def coordinateInInterval = source.getWeather(timeInterval, invalidCoordinate) + when: "requesting weather for an invalid coordinate at a specific date" + source.getWeather(time, invalidCoordinate) + + then: "NoDataException is thrown" + def ex1 = thrown(NoDataException) + ex1.message.contains("No coordinate ID found for the given point") + ex1.message.contains(invalidCoordinate.toString()) + + when: "requesting weather for an invalid coordinate in a time interval" + source.getWeather(timeInterval, invalidCoordinate) + + then: "NoDataException is thrown" + def ex2 = thrown(NoDataException) + ex2.message.contains("No data for given coordinates") + ex2.message.contains(invalidCoordinate.toString()) + + when: "requesting weather for mixed valid and invalid coordinates" def coordinatesToTimeSeries = source.getWeather(timeInterval, [ validCoordinate, invalidCoordinate ]) - then: - assert coordinateAtDate == null - equalsIgnoreUUID(coordinateInInterval, emptyTimeSeries) + then: "only valid coordinate data is returned" coordinatesToTimeSeries.keySet() == [validCoordinate].toSet() equalsIgnoreUUID(coordinatesToTimeSeries.get(validCoordinate), timeseries_193186) } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy index e4d62131e..47c4b086a 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source.influxdb +import edu.ie3.datamodel.exceptions.NoDataException import edu.ie3.datamodel.io.connectors.InfluxDbConnector import edu.ie3.datamodel.io.factory.timeseries.IconTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries @@ -118,13 +119,12 @@ class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSource equalsIgnoreUUID(coordinateToTimeSeries.get(IconWeatherTestData.COORDINATE_67776).entries, timeseries67776.entries) } - def "An InfluxDbWeatherSource will return an equivalent to 'empty' when being unable to map a coordinate to its ID"() { + def "An InfluxDbWeatherSource will throw NoDataException when being unable to map a coordinate to its ID"() { given: def validCoordinate = IconWeatherTestData.COORDINATE_67775 def invalidCoordinate = GeoUtils.buildPoint(7d, 48d) def time = IconWeatherTestData.TIME_15H def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_15H , IconWeatherTestData.TIME_17H) - def emptyTimeSeries = new IndividualTimeSeries(UUID.randomUUID(), Collections.emptySet()) def timeseries67775 = new IndividualTimeSeries(null, [ new TimeBasedValue(IconWeatherTestData.TIME_15H, IconWeatherTestData.WEATHER_VALUE_67775_15H), @@ -132,17 +132,29 @@ class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSource new TimeBasedValue(IconWeatherTestData.TIME_17H, IconWeatherTestData.WEATHER_VALUE_67775_17H) ] as Set) - when: - def coordinateAtDate = source.getWeather(time, invalidCoordinate) - def coordinateInInterval = source.getWeather(timeInterval, invalidCoordinate) + when: "requesting weather for an invalid coordinate at a specific date" + source.getWeather(time, invalidCoordinate) + + then: "NoDataException is thrown" + def ex1 = thrown(NoDataException) + ex1.message.contains("No coordinate ID found for the given point") + ex1.message.contains(invalidCoordinate.toString()) + + when: "requesting weather for an invalid coordinate in a time interval" + source.getWeather(timeInterval, invalidCoordinate) + + then: "NoDataException is thrown" + def ex2 = thrown(NoDataException) + ex2.message.contains("No data for given coordinates") + ex2.message.contains(invalidCoordinate.toString()) + + when: "requesting weather for mixed valid and invalid coordinates" def coordinatesToTimeSeries = source.getWeather(timeInterval, [ validCoordinate, invalidCoordinate ]) - then: - assert coordinateAtDate == null - equalsIgnoreUUID(coordinateInInterval, emptyTimeSeries) + then: "only valid coordinate data is returned" coordinatesToTimeSeries.keySet() == [validCoordinate].toSet() equalsIgnoreUUID(coordinatesToTimeSeries.get(validCoordinate), timeseries67775) } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index a903a2299..4b8787f5c 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source.sql +import edu.ie3.datamodel.exceptions.NoDataException import edu.ie3.datamodel.io.connectors.SqlConnector import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries @@ -60,12 +61,17 @@ class SqlWeatherSourceCosmoIT extends Specification implements TestContainerHelp equalsIgnoreUUID(optTimeBasedValue, expectedTimeBasedValue ) } - def "A SqlWeatherSource returns nothing for an invalid coordinate"() { + def "A SqlWeatherSource throws NoDataException for an invalid coordinate"() { + given: + def invalidCoordinate = GeoUtils.buildPoint(89d, 88d) + when: - def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, GeoUtils.buildPoint(89d, 88d)) + source.getWeather(CosmoWeatherTestData.TIME_15H, invalidCoordinate) then: - assert optTimeBasedValue == null + def ex = thrown(NoDataException) + ex.message.contains("No coordinate ID found for the given point") + ex.message.contains(invalidCoordinate.toString()) } def "A SqlWeatherSource can read multiple time series values for multiple coordinates"() { From 35a497d551c5261f183a19c2aab6418f2f4cdd96 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 20 Aug 2025 23:47:04 +0200 Subject: [PATCH 27/53] spotless --- .../io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy | 2 +- .../io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy index 7da8b157b..047c64f3e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy @@ -147,7 +147,7 @@ class InfluxDbWeatherSourceCosmoIT extends Specification implements TestContaine ex1.message.contains("No coordinate ID found for the given point") ex1.message.contains(invalidCoordinate.toString()) - when: "requesting weather for an invalid coordinate in a time interval" + when: "requesting weather for an invalid coordinate in a time interval" source.getWeather(timeInterval, invalidCoordinate) then: "NoDataException is thrown" diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy index 47c4b086a..e8b57066a 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy @@ -140,7 +140,7 @@ class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSource ex1.message.contains("No coordinate ID found for the given point") ex1.message.contains(invalidCoordinate.toString()) - when: "requesting weather for an invalid coordinate in a time interval" + when: "requesting weather for an invalid coordinate in a time interval" source.getWeather(timeInterval, invalidCoordinate) then: "NoDataException is thrown" From 3951355ea0669960e18fe666d139a8a666a8f870 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 20 Aug 2025 23:54:23 +0200 Subject: [PATCH 28/53] tests --- .../source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy | 8 +++----- .../io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy | 8 +++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy index 047c64f3e..737072bf9 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy @@ -148,12 +148,10 @@ class InfluxDbWeatherSourceCosmoIT extends Specification implements TestContaine ex1.message.contains(invalidCoordinate.toString()) when: "requesting weather for an invalid coordinate in a time interval" - source.getWeather(timeInterval, invalidCoordinate) + def result = source.getWeather(timeInterval, invalidCoordinate) - then: "NoDataException is thrown" - def ex2 = thrown(NoDataException) - ex2.message.contains("No data for given coordinates") - ex2.message.contains(invalidCoordinate.toString()) + then: "empty time series is returned" + result.entries.isEmpty() when: "requesting weather for mixed valid and invalid coordinates" def coordinatesToTimeSeries = source.getWeather(timeInterval, [ diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy index e8b57066a..a53831e6c 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy @@ -141,12 +141,10 @@ class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSource ex1.message.contains(invalidCoordinate.toString()) when: "requesting weather for an invalid coordinate in a time interval" - source.getWeather(timeInterval, invalidCoordinate) + def result = source.getWeather(timeInterval, invalidCoordinate) - then: "NoDataException is thrown" - def ex2 = thrown(NoDataException) - ex2.message.contains("No data for given coordinates") - ex2.message.contains(invalidCoordinate.toString()) + then: "empty time series is returned" + result.entries.isEmpty() when: "requesting weather for mixed valid and invalid coordinates" def coordinatesToTimeSeries = source.getWeather(timeInterval, [ From 24ff4b51d68cd714e978a98fd38e4a333c335181 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Thu, 21 Aug 2025 00:14:30 +0200 Subject: [PATCH 29/53] added exception to couchbase tests --- .../CouchbaseWeatherSourceCosmoIT.groovy | 27 +++++++++++++++++++ .../CouchbaseWeatherSourceIconIT.groovy | 27 +++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index 07ffa0f03..952d57a36 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source.couchbase +import edu.ie3.datamodel.exceptions.NoDataException import edu.ie3.datamodel.io.connectors.CouchbaseConnector import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries @@ -13,6 +14,7 @@ import edu.ie3.datamodel.models.value.WeatherValue import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.test.helper.TestContainerHelper import edu.ie3.test.helper.WeatherSourceTestHelper +import edu.ie3.util.geo.GeoUtils import edu.ie3.util.interval.ClosedInterval import org.locationtech.jts.geom.Point import org.testcontainers.couchbase.BucketDefinition @@ -164,4 +166,29 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements TestContain ] actual.get(CosmoWeatherTestData.COORDINATE_193187) == [CosmoWeatherTestData.TIME_16H] } + + def "A CouchbaseWeatherSource throws NoDataException for invalid coordinate"() { + given: + def invalidCoordinate = GeoUtils.buildPoint(999d, 999d) + + when: + source.getWeather(CosmoWeatherTestData.TIME_15H, invalidCoordinate) + + then: + def ex = thrown(NoDataException) + ex.message.contains("No coordinate ID found for the given point") + ex.message.contains(invalidCoordinate.toString()) + } + + def "A CouchbaseWeatherSource throws NoDataException for future date"() { + given: + def futureDate = CosmoWeatherTestData.TIME_17H.plusDays(30) + + when: + source.getWeather(futureDate, CosmoWeatherTestData.COORDINATE_193186) + + then: + def ex = thrown(NoDataException) + ex.message.contains("Weather document not found") + } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy index d83d37a2e..a08cf5223 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source.couchbase +import edu.ie3.datamodel.exceptions.NoDataException import edu.ie3.datamodel.io.connectors.CouchbaseConnector import edu.ie3.datamodel.io.factory.timeseries.IconTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries @@ -13,6 +14,7 @@ import edu.ie3.test.common.IconWeatherTestData import edu.ie3.test.helper.TestContainerHelper import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.TimeUtil +import edu.ie3.util.geo.GeoUtils import edu.ie3.util.interval.ClosedInterval import org.testcontainers.couchbase.BucketDefinition import org.testcontainers.couchbase.CouchbaseContainer @@ -161,4 +163,29 @@ class CouchbaseWeatherSourceIconIT extends Specification implements TestContaine ] actual.get(IconWeatherTestData.COORDINATE_67776) == [IconWeatherTestData.TIME_16H] } + + def "A CouchbaseWeatherSource throws NoDataException for invalid coordinate"() { + given: + def invalidCoordinate = GeoUtils.buildPoint(999d, 999d) + + when: + source.getWeather(IconWeatherTestData.TIME_15H, invalidCoordinate) + + then: + def ex = thrown(NoDataException) + ex.message.contains("No coordinate ID found for the given point") + ex.message.contains(invalidCoordinate.toString()) + } + + def "A CouchbaseWeatherSource throws NoDataException for future date"() { + given: + def futureDate = IconWeatherTestData.TIME_17H.plusDays(30) + + when: + source.getWeather(futureDate, IconWeatherTestData.COORDINATE_67775) + + then: + def ex = thrown(NoDataException) + ex.message.contains("Weather document not found") + } } From e8a4fee5312cce9b800bedf9d6fb2f9a4fd9080f Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Thu, 21 Aug 2025 00:31:54 +0200 Subject: [PATCH 30/53] some fixes --- .../source/couchbase/CouchbaseWeatherSource.java | 9 ++++++++- .../io/source/influxdb/InfluxDbWeatherSource.java | 12 +++--------- .../datamodel/io/source/sql/SqlWeatherSource.java | 14 ++++---------- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index caf19023f..4719801c3 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -175,7 +175,14 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin } catch (CompletionException ex) { if (ex.getCause() instanceof DocumentNotFoundException) { throw new NoDataException("Weather document not found in the completion stage."); - } else throw ex; + } else { + logger.error( + "Unexpected completion exception while retrieving weather data for coordinate {} and date {}", + coordinate, + date, + ex); + throw ex; + } } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 9b2f11171..15cbffc34 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -124,15 +124,9 @@ public Map> getWeather( @Override public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws NoDataException { - Optional coordinateId; - try { - coordinateId = idCoordinateSource.getId(coordinate); - if (coordinateId.isEmpty()) { - throw new NoDataException("No coordinate ID found for the given point: " + coordinate); - } - } catch (NoDataException e) { - log.error("No data available for coordinate {} and date {}", coordinate, date, e); - throw e; + Optional coordinateId = idCoordinateSource.getId(coordinate); + if (coordinateId.isEmpty()) { + throw new NoDataException("No coordinate ID found for the given point: " + coordinate); } try (InfluxDB session = connector.getSession()) { diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 8955864f9..81bcebb83 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -148,16 +148,10 @@ public Map> getWeather( @Override public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws SourceException, NoDataException { - Optional coordinateId; - try { - coordinateId = idCoordinateSource.getId(coordinate); - if (coordinateId.isEmpty()) { - log.warn("Unable to match coordinate {} to a coordinate ID", coordinate); - throw new NoDataException("No coordinate ID found for the given point: " + coordinate); - } - } catch (NoDataException e) { - log.error("No data available for coordinate {} and date {}", coordinate, date, e); - throw e; + Optional coordinateId = idCoordinateSource.getId(coordinate); + if (coordinateId.isEmpty()) { + log.warn("Unable to match coordinate {} to a coordinate ID", coordinate); + throw new NoDataException("No coordinate ID found for the given point: " + coordinate); } List> timeBasedValues = From 5cc104b76a496ee9baee48ffad8b8d414ce3b88a Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Thu, 21 Aug 2025 00:46:13 +0200 Subject: [PATCH 31/53] some fixes --- .../io/source/couchbase/CouchbaseWeatherSource.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 4719801c3..c6feae692 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -168,9 +168,14 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin new NoDataException( "No valid weather data found for the given date and coordinate.")); } catch (DecodingFailureException ex) { - logger.error("Decoding to TimeBasedWeatherValue failed!", ex); + logger.error( + "Decoding to TimeBasedWeatherValue failed for coordinate {} and date {}", + coordinate, + date, + ex); throw new NoDataException("Failed to decode weather data: " + ex.getMessage()); } catch (DocumentNotFoundException ex) { + logger.warn("Weather document not found for coordinate {} and date {}", coordinate, date); throw new NoDataException("Weather document not found."); } catch (CompletionException ex) { if (ex.getCause() instanceof DocumentNotFoundException) { From 9f97b00dbce3f9bd2aaf0cdf32bbc074b3841b3d Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Thu, 21 Aug 2025 00:55:27 +0200 Subject: [PATCH 32/53] no more missing data --- .../couchbase/CouchbaseWeatherSource.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index c6feae692..e8f880af0 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -173,13 +173,24 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin coordinate, date, ex); - throw new NoDataException("Failed to decode weather data: " + ex.getMessage()); + throw new NoDataException( + "Failed to decode weather data for coordinate " + + coordinate + + " and date " + + date + + ": " + + ex.getMessage()); } catch (DocumentNotFoundException ex) { logger.warn("Weather document not found for coordinate {} and date {}", coordinate, date); - throw new NoDataException("Weather document not found."); + throw new NoDataException( + "Weather document not found for coordinate " + coordinate + " and date " + date); } catch (CompletionException ex) { if (ex.getCause() instanceof DocumentNotFoundException) { - throw new NoDataException("Weather document not found in the completion stage."); + throw new NoDataException( + "Weather document not found in the completion stage for coordinate " + + coordinate + + " and date " + + date); } else { logger.error( "Unexpected completion exception while retrieving weather data for coordinate {} and date {}", From 4be6590320a04a80ebab1b2d13182f407d7b252f Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Thu, 21 Aug 2025 01:03:17 +0200 Subject: [PATCH 33/53] edited exception --- .../java/edu/ie3/datamodel/exceptions/NoDataException.java | 4 ++++ .../datamodel/io/source/couchbase/CouchbaseWeatherSource.java | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java index dc52fcb6a..d7004c8a4 100644 --- a/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java +++ b/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java @@ -17,4 +17,8 @@ public class NoDataException extends Exception { public NoDataException(final String message) { super(message); } + + public NoDataException(final String message, final Throwable cause) { + super(message, cause); + } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index e8f880af0..827f9149a 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -179,7 +179,8 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin + " and date " + date + ": " - + ex.getMessage()); + + ex.getMessage(), + ex); } catch (DocumentNotFoundException ex) { logger.warn("Weather document not found for coordinate {} and date {}", coordinate, date); throw new NoDataException( From f20db541f952fc48a2540d96ea9e81fbcdac0663 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Thu, 21 Aug 2025 01:16:12 +0200 Subject: [PATCH 34/53] bugfix --- .../couchbase/CouchbaseWeatherSource.java | 34 +++++++------------ 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 827f9149a..4f5331d90 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -168,37 +168,27 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin new NoDataException( "No valid weather data found for the given date and coordinate.")); } catch (DecodingFailureException ex) { - logger.error( - "Decoding to TimeBasedWeatherValue failed for coordinate {} and date {}", - coordinate, - date, - ex); throw new NoDataException( - "Failed to decode weather data for coordinate " - + coordinate - + " and date " - + date - + ": " - + ex.getMessage(), - ex); + "Failed to decode weather data for coordinate " + coordinate + " and date " + date, ex); } catch (DocumentNotFoundException ex) { - logger.warn("Weather document not found for coordinate {} and date {}", coordinate, date); throw new NoDataException( - "Weather document not found for coordinate " + coordinate + " and date " + date); + "Weather document not found for coordinate " + coordinate + " and date " + date, ex); } catch (CompletionException ex) { - if (ex.getCause() instanceof DocumentNotFoundException) { + Throwable cause = ex.getCause(); + if (cause instanceof DocumentNotFoundException) { throw new NoDataException( - "Weather document not found in the completion stage for coordinate " + "Weather document not found in completion stage for coordinate " + coordinate + " and date " - + date); + + date, + cause); } else { - logger.error( - "Unexpected completion exception while retrieving weather data for coordinate {} and date {}", - coordinate, - date, + throw new NoDataException( + "Unexpected completion exception while retrieving weather data for coordinate " + + coordinate + + " and date " + + date, ex); - throw ex; } } } From fd0823be8fbe9680498f7e10cb745ecf5a3a0528 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Thu, 21 Aug 2025 02:06:12 +0200 Subject: [PATCH 35/53] fx --- .../datamodel/io/source/couchbase/CouchbaseWeatherSource.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 4f5331d90..4077e9b3a 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -128,7 +128,9 @@ public Map> getWeather( try { jsonWeatherInputs = queryResult.rowsAsObject(); } catch (DecodingFailureException ex) { - logger.error("Querying weather inputs failed!", ex); + logger.error("Querying weather inputs failed for coordinate {}", coordinate, ex); + throw new NoDataException( + "Failed to decode weather data for coordinate " + coordinate, ex); } if (jsonWeatherInputs != null && !jsonWeatherInputs.isEmpty()) { Set> weatherInputs = From ef08123e13bc44050f2b5b2cce7cac548729f34c Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Thu, 21 Aug 2025 02:12:04 +0200 Subject: [PATCH 36/53] redundant log removed --- .../datamodel/io/source/couchbase/CouchbaseWeatherSource.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 4077e9b3a..6bdd9b76b 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -128,7 +128,6 @@ public Map> getWeather( try { jsonWeatherInputs = queryResult.rowsAsObject(); } catch (DecodingFailureException ex) { - logger.error("Querying weather inputs failed for coordinate {}", coordinate, ex); throw new NoDataException( "Failed to decode weather data for coordinate " + coordinate, ex); } From cf8adb3242f522fc520bc5b6f5fc41f9d3422514 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Thu, 21 Aug 2025 02:47:36 +0200 Subject: [PATCH 37/53] hopefully final --- .../influxdb/InfluxDbWeatherSource.java | 29 +++++-------------- .../io/source/sql/SqlWeatherSource.java | 2 -- .../InfluxDbWeatherSourceCosmoIT.groovy | 8 +++-- .../InfluxDbWeatherSourceIconIT.groovy | 8 +++-- 4 files changed, 18 insertions(+), 29 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 15cbffc34..5dae98f7b 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -81,15 +81,12 @@ public Map> getWeather( return coordinateToValues.entrySet().stream() .collect( Collectors.toMap(Map.Entry::getKey, e -> new IndividualTimeSeries<>(e.getValue()))); - } catch (NoDataException e) { - log.error("No data available for coordinate", e); - return Collections.emptyMap(); } } @Override public Map> getWeather( - ClosedInterval timeInterval, Collection coordinates) { + ClosedInterval timeInterval, Collection coordinates) throws NoDataException { if (coordinates == null) return getWeather(timeInterval); Map> coordinatesToId = coordinates.stream().collect(Collectors.toMap(point -> point, idCoordinateSource::getId)); @@ -105,18 +102,13 @@ public Map> getWeather( optTimeBasedValueStream(queryResult); Set> timeBasedValues = filterEmptyOptionals(optValues).collect(Collectors.toSet()); - if (timeBasedValues.isEmpty()) { - throw new NoDataException( - "No weather data found for the given time interval and coordinate: " - + entry.getKey()); + if (!timeBasedValues.isEmpty()) { + IndividualTimeSeries timeSeries = + new IndividualTimeSeries<>(timeBasedValues); + coordinateToTimeSeries.put(entry.getKey(), timeSeries); } - IndividualTimeSeries timeSeries = - new IndividualTimeSeries<>(timeBasedValues); - coordinateToTimeSeries.put(entry.getKey(), timeSeries); } } - } catch (NoDataException e) { - return Collections.emptyMap(); } return coordinateToTimeSeries; } @@ -193,14 +185,9 @@ public List getTimeKeysAfter(ZonedDateTime time, Point coordinate */ public IndividualTimeSeries getWeather( ClosedInterval timeInterval, Point coordinate) throws NoDataException { - Optional coordinateId; - try { - coordinateId = idCoordinateSource.getId(coordinate); - if (coordinateId.isEmpty()) { - throw new NoDataException("No data for given coordinates: " + coordinate); - } - } catch (NoDataException e) { - return new IndividualTimeSeries<>(UUID.randomUUID(), Collections.emptySet()); + Optional coordinateId = idCoordinateSource.getId(coordinate); + if (coordinateId.isEmpty()) { + throw new NoDataException("No data for given coordinates: " + coordinate); } try (InfluxDB session = connector.getSession()) { String query = diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 81bcebb83..8928b12b8 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -125,8 +125,6 @@ public Map> getWeather( log.warn("Unable to match coordinates to coordinate ID"); throw new NoDataException("No coordinates found"); } - } catch (NoDataException e) { - return Collections.emptyMap(); } List> timeBasedValues = diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy index 737072bf9..047c64f3e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy @@ -148,10 +148,12 @@ class InfluxDbWeatherSourceCosmoIT extends Specification implements TestContaine ex1.message.contains(invalidCoordinate.toString()) when: "requesting weather for an invalid coordinate in a time interval" - def result = source.getWeather(timeInterval, invalidCoordinate) + source.getWeather(timeInterval, invalidCoordinate) - then: "empty time series is returned" - result.entries.isEmpty() + then: "NoDataException is thrown" + def ex2 = thrown(NoDataException) + ex2.message.contains("No data for given coordinates") + ex2.message.contains(invalidCoordinate.toString()) when: "requesting weather for mixed valid and invalid coordinates" def coordinatesToTimeSeries = source.getWeather(timeInterval, [ diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy index a53831e6c..e8b57066a 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy @@ -141,10 +141,12 @@ class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSource ex1.message.contains(invalidCoordinate.toString()) when: "requesting weather for an invalid coordinate in a time interval" - def result = source.getWeather(timeInterval, invalidCoordinate) + source.getWeather(timeInterval, invalidCoordinate) - then: "empty time series is returned" - result.entries.isEmpty() + then: "NoDataException is thrown" + def ex2 = thrown(NoDataException) + ex2.message.contains("No data for given coordinates") + ex2.message.contains(invalidCoordinate.toString()) when: "requesting weather for mixed valid and invalid coordinates" def coordinatesToTimeSeries = source.getWeather(timeInterval, [ From 701d7e23096ca09d8097375e7d62ebe3638a0f14 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Thu, 21 Aug 2025 02:55:04 +0200 Subject: [PATCH 38/53] spotless --- .../influxdb/InfluxDbWeatherSource.java | 3 ++- .../io/source/sql/SqlWeatherSource.java | 19 ++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 5dae98f7b..310887b7d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -86,7 +86,8 @@ public Map> getWeather( @Override public Map> getWeather( - ClosedInterval timeInterval, Collection coordinates) throws NoDataException { + ClosedInterval timeInterval, Collection coordinates) + throws NoDataException { if (coordinates == null) return getWeather(timeInterval); Map> coordinatesToId = coordinates.stream().collect(Collectors.toMap(point -> point, idCoordinateSource::getId)); diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 8928b12b8..c1d92c87e 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -114,17 +114,14 @@ public Map> getWeather( public Map> getWeather( ClosedInterval timeInterval, Collection coordinates) throws SourceException, NoDataException { - Set coordinateIds; - try { - coordinateIds = - coordinates.stream() - .map(idCoordinateSource::getId) - .flatMap(Optional::stream) - .collect(Collectors.toSet()); - if (coordinateIds.isEmpty()) { - log.warn("Unable to match coordinates to coordinate ID"); - throw new NoDataException("No coordinates found"); - } + Set coordinateIds = + coordinates.stream() + .map(idCoordinateSource::getId) + .flatMap(Optional::stream) + .collect(Collectors.toSet()); + if (coordinateIds.isEmpty()) { + log.warn("Unable to match coordinates to coordinate ID"); + throw new NoDataException("No coordinates found"); } List> timeBasedValues = From b5d2ccfd244f84a63f10e9d6ce0071a19dcf8bbf Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Thu, 21 Aug 2025 02:56:04 +0200 Subject: [PATCH 39/53] more information --- .../edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index c1d92c87e..6fdccde03 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -120,8 +120,8 @@ public Map> getWeather( .flatMap(Optional::stream) .collect(Collectors.toSet()); if (coordinateIds.isEmpty()) { - log.warn("Unable to match coordinates to coordinate ID"); - throw new NoDataException("No coordinates found"); + log.warn("Unable to match coordinates {} to coordinate IDs", coordinates); + throw new NoDataException("Unable to match any of the provided coordinates to coordinate IDs: " + coordinates); } List> timeBasedValues = From dfb49df2e0584e936ba932ac1b4f44c24af6eb9a Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Thu, 21 Aug 2025 03:00:48 +0200 Subject: [PATCH 40/53] fix --- .../ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 310887b7d..6d70d5e06 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -60,7 +60,7 @@ public Optional> getSourceFields() { @Override public Map> getWeather( - ClosedInterval timeInterval) { + ClosedInterval timeInterval) throws NoDataException { try (InfluxDB session = connector.getSession()) { String query = createQueryStringForTimeInterval(timeInterval); QueryResult queryResult = session.query(new Query(query)); From c75b9fafc05619c06ec83a5d3a730135a25d7bf7 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Thu, 21 Aug 2025 03:01:47 +0200 Subject: [PATCH 41/53] aa --- .../java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 6fdccde03..3c6bb5eaf 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -121,7 +121,8 @@ public Map> getWeather( .collect(Collectors.toSet()); if (coordinateIds.isEmpty()) { log.warn("Unable to match coordinates {} to coordinate IDs", coordinates); - throw new NoDataException("Unable to match any of the provided coordinates to coordinate IDs: " + coordinates); + throw new NoDataException( + "Unable to match any of the provided coordinates to coordinate IDs: " + coordinates); } List> timeBasedValues = From dc354340a9f5c86f06e3baeab2f7189a2334019e Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Thu, 21 Aug 2025 03:14:13 +0200 Subject: [PATCH 42/53] Revert "redundant log removed" This reverts commit ef08123e13bc44050f2b5b2cce7cac548729f34c. --- .../datamodel/io/source/couchbase/CouchbaseWeatherSource.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 6bdd9b76b..4077e9b3a 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -128,6 +128,7 @@ public Map> getWeather( try { jsonWeatherInputs = queryResult.rowsAsObject(); } catch (DecodingFailureException ex) { + logger.error("Querying weather inputs failed for coordinate {}", coordinate, ex); throw new NoDataException( "Failed to decode weather data for coordinate " + coordinate, ex); } From 0800310ab9552a5c418a0135f54e82afa851a7ce Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Thu, 21 Aug 2025 03:20:14 +0200 Subject: [PATCH 43/53] Revert "fx" This reverts commit fd0823be8fbe9680498f7e10cb745ecf5a3a0528. --- .../datamodel/io/source/couchbase/CouchbaseWeatherSource.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 4077e9b3a..4f5331d90 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -128,9 +128,7 @@ public Map> getWeather( try { jsonWeatherInputs = queryResult.rowsAsObject(); } catch (DecodingFailureException ex) { - logger.error("Querying weather inputs failed for coordinate {}", coordinate, ex); - throw new NoDataException( - "Failed to decode weather data for coordinate " + coordinate, ex); + logger.error("Querying weather inputs failed!", ex); } if (jsonWeatherInputs != null && !jsonWeatherInputs.isEmpty()) { Set> weatherInputs = From a2911b0149c53c4629297dcbb9a6f6548b4555bd Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Thu, 21 Aug 2025 03:28:19 +0200 Subject: [PATCH 44/53] sql --- .../io/source/sql/SqlWeatherSourceCosmoIT.groovy | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index 4b8787f5c..8bd0d697c 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -101,7 +101,7 @@ class SqlWeatherSourceCosmoIT extends Specification implements TestContainerHelp equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeSeries193187) } - def "A SqlWeatherSource returns nothing for invalid coordinates"() { + def "A SqlWeatherSource throws NoDataException for invalid coordinates"() { given: def coordinates = [ GeoUtils.buildPoint(89d, 88d), @@ -110,10 +110,12 @@ class SqlWeatherSourceCosmoIT extends Specification implements TestContainerHelp def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.TIME_17H) when: - Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) + source.getWeather(timeInterval, coordinates) then: - coordinateToTimeSeries == Collections.emptyMap() + def ex = thrown(NoDataException) + ex.message.contains("Unable to match any of the provided coordinates to coordinate IDs") + ex.message.contains(coordinates.toString()) } def "A SqlWeatherSource can read all weather data in a given time interval"() { From ce294c5eb1e0819fbf6ee36049fb1d081d36865e Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Thu, 21 Aug 2025 03:32:02 +0200 Subject: [PATCH 45/53] couchbaase --- .../datamodel/io/source/couchbase/CouchbaseWeatherSource.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 4f5331d90..6bdd9b76b 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -128,7 +128,8 @@ public Map> getWeather( try { jsonWeatherInputs = queryResult.rowsAsObject(); } catch (DecodingFailureException ex) { - logger.error("Querying weather inputs failed!", ex); + throw new NoDataException( + "Failed to decode weather data for coordinate " + coordinate, ex); } if (jsonWeatherInputs != null && !jsonWeatherInputs.isEmpty()) { Set> weatherInputs = From c38e7a54915aa183767030f4705c999e88d7acd6 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 27 Aug 2025 01:49:59 +0200 Subject: [PATCH 46/53] hopefully final --- .../couchbase/CouchbaseWeatherSource.java | 14 ++++++++++---- .../io/source/csv/CsvWeatherSource.java | 10 ++++++++++ .../source/influxdb/InfluxDbWeatherSource.java | 11 +++++++++++ .../io/source/sql/SqlWeatherSource.java | 16 +++++++++++----- .../CouchbaseWeatherSourceCosmoIT.groovy | 18 ++++++++++++++++++ .../CouchbaseWeatherSourceIconIT.groovy | 18 ++++++++++++++++++ .../source/csv/CsvWeatherSourceIconTest.groovy | 18 ++++++++++++++++++ .../InfluxDbWeatherSourceCosmoIT.groovy | 9 +++++---- .../InfluxDbWeatherSourceIconIT.groovy | 9 +++++---- .../source/sql/SqlWeatherSourceCosmoIT.groovy | 18 ++++++++++++++++++ 10 files changed, 124 insertions(+), 17 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 6bdd9b76b..00761284b 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -117,6 +117,16 @@ public Map> getWeather( public Map> getWeather( ClosedInterval timeInterval, Collection coordinates) throws NoDataException { + + List invalidCoordinates = + coordinates.stream() + .filter(coordinate -> idCoordinateSource.getId(coordinate).isEmpty()) + .toList(); + + if (!invalidCoordinates.isEmpty()) { + throw new NoDataException("No data for given coordinates: " + invalidCoordinates); + } + HashMap> coordinateToTimeSeries = new HashMap<>(); for (Point coordinate : coordinates) { Optional coordinateId = idCoordinateSource.getId(coordinate); @@ -141,10 +151,6 @@ public Map> getWeather( new IndividualTimeSeries<>(weatherInputs); coordinateToTimeSeries.put(coordinate, weatherTimeSeries); } - } else { - logger.error("Unable to match coordinate {} to a coordinate ID", coordinate); - throw new NoDataException( - "Unable to match coordinate " + coordinate + " to a coordinate ID"); } } return coordinateToTimeSeries; diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index d0aef8a06..45035808d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -108,6 +108,16 @@ public Map> getWeather( public Map> getWeather( ClosedInterval timeInterval, Collection coordinates) throws NoDataException { + + List invalidCoordinates = + coordinates.stream() + .filter(coordinate -> !coordinateToTimeSeries.containsKey(coordinate)) + .toList(); + + if (!invalidCoordinates.isEmpty()) { + throw new NoDataException("No data for given coordinates: " + invalidCoordinates); + } + Map> filteredMap = coordinateToTimeSeries.entrySet().stream() .filter(entry -> coordinates.contains(entry.getKey())) diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 6d70d5e06..85f0a2d2a 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -91,6 +91,17 @@ public Map> getWeather( if (coordinates == null) return getWeather(timeInterval); Map> coordinatesToId = coordinates.stream().collect(Collectors.toMap(point -> point, idCoordinateSource::getId)); + + List invalidCoordinates = + coordinatesToId.entrySet().stream() + .filter(entry -> entry.getValue().isEmpty()) + .map(Map.Entry::getKey) + .toList(); + + if (!invalidCoordinates.isEmpty()) { + throw new NoDataException("No data for given coordinates: " + invalidCoordinates); + } + HashMap> coordinateToTimeSeries = new HashMap<>(); try (InfluxDB session = connector.getSession()) { for (Map.Entry> entry : coordinatesToId.entrySet()) { diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 3c6bb5eaf..30dd703ed 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -114,16 +114,22 @@ public Map> getWeather( public Map> getWeather( ClosedInterval timeInterval, Collection coordinates) throws SourceException, NoDataException { + + List invalidCoordinates = + coordinates.stream() + .filter(coordinate -> idCoordinateSource.getId(coordinate).isEmpty()) + .collect(Collectors.toList()); + + if (!invalidCoordinates.isEmpty()) { + log.warn("Unable to match coordinates {} to coordinate IDs", invalidCoordinates); + throw new NoDataException("No data for given coordinates: " + invalidCoordinates); + } + Set coordinateIds = coordinates.stream() .map(idCoordinateSource::getId) .flatMap(Optional::stream) .collect(Collectors.toSet()); - if (coordinateIds.isEmpty()) { - log.warn("Unable to match coordinates {} to coordinate IDs", coordinates); - throw new NoDataException( - "Unable to match any of the provided coordinates to coordinate IDs: " + coordinates); - } List> timeBasedValues = buildTimeBasedValues( diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index 952d57a36..5b35371b2 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -180,6 +180,24 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements TestContain ex.message.contains(invalidCoordinate.toString()) } + def "A CouchbaseWeatherSource throws NoDataException for mixed valid and invalid coordinates"() { + given: + def validCoordinate = CosmoWeatherTestData.COORDINATE_193186 + def invalidCoordinate = GeoUtils.buildPoint(999d, 999d) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) + + when: + source.getWeather(timeInterval, [ + validCoordinate, + invalidCoordinate + ]) + + then: + def ex = thrown(NoDataException) + ex.message.contains("No data for given coordinates") + ex.message.contains(invalidCoordinate.toString()) + } + def "A CouchbaseWeatherSource throws NoDataException for future date"() { given: def futureDate = CosmoWeatherTestData.TIME_17H.plusDays(30) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy index a08cf5223..8dbdcf8db 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy @@ -177,6 +177,24 @@ class CouchbaseWeatherSourceIconIT extends Specification implements TestContaine ex.message.contains(invalidCoordinate.toString()) } + def "A CouchbaseWeatherSource throws NoDataException for mixed valid and invalid coordinates"() { + given: + def validCoordinate = IconWeatherTestData.COORDINATE_67775 + def invalidCoordinate = GeoUtils.buildPoint(999d, 999d) + def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_15H, IconWeatherTestData.TIME_17H) + + when: + source.getWeather(timeInterval, [ + validCoordinate, + invalidCoordinate + ]) + + then: + def ex = thrown(NoDataException) + ex.message.contains("No data for given coordinates") + ex.message.contains(invalidCoordinate.toString()) + } + def "A CouchbaseWeatherSource throws NoDataException for future date"() { given: def futureDate = IconWeatherTestData.TIME_17H.plusDays(30) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy index 18677c887..7e3f22f8e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy @@ -322,4 +322,22 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, ex.message.contains("No weather data found for the given coordinate") ex.message.contains(IconWeatherTestData.COORDINATE_67775.toString()) } + + def "A CsvWeatherSource throws NoDataException for mixed valid and invalid coordinates"() { + given: + def validCoordinate = IconWeatherTestData.COORDINATE_67775 + def invalidCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(999d, 999d)) + def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_15H, IconWeatherTestData.TIME_17H) + + when: + source.getWeather(timeInterval, [ + validCoordinate, + invalidCoordinate + ]) + + then: + def ex = thrown(NoDataException) + ex.message.contains("No data for given coordinates") + ex.message.contains(invalidCoordinate.toString()) + } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy index 047c64f3e..28e6a7d81 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy @@ -156,14 +156,15 @@ class InfluxDbWeatherSourceCosmoIT extends Specification implements TestContaine ex2.message.contains(invalidCoordinate.toString()) when: "requesting weather for mixed valid and invalid coordinates" - def coordinatesToTimeSeries = source.getWeather(timeInterval, [ + source.getWeather(timeInterval, [ validCoordinate, invalidCoordinate ]) - then: "only valid coordinate data is returned" - coordinatesToTimeSeries.keySet() == [validCoordinate].toSet() - equalsIgnoreUUID(coordinatesToTimeSeries.get(validCoordinate), timeseries_193186) + then: "NoDataException is thrown" + def ex3 = thrown(NoDataException) + ex3.message.contains("No data for given coordinates") + ex3.message.contains(invalidCoordinate.toString()) } def "A InfluxDbWeatherSource returns all time keys after a given time key correctly"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy index e8b57066a..5bee62a39 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy @@ -149,14 +149,15 @@ class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSource ex2.message.contains(invalidCoordinate.toString()) when: "requesting weather for mixed valid and invalid coordinates" - def coordinatesToTimeSeries = source.getWeather(timeInterval, [ + source.getWeather(timeInterval, [ validCoordinate, invalidCoordinate ]) - then: "only valid coordinate data is returned" - coordinatesToTimeSeries.keySet() == [validCoordinate].toSet() - equalsIgnoreUUID(coordinatesToTimeSeries.get(validCoordinate), timeseries67775) + then: "NoDataException is thrown" + def ex3 = thrown(NoDataException) + ex3.message.contains("No data for given coordinates") + ex3.message.contains(invalidCoordinate.toString()) } def "The InfluxDbWeatherSource returns all time keys after a given time key correctly"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index 8bd0d697c..55bb3a555 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -163,4 +163,22 @@ class SqlWeatherSourceCosmoIT extends Specification implements TestContainerHelp ] actual.get(CosmoWeatherTestData.COORDINATE_193187) == [CosmoWeatherTestData.TIME_16H] } + + def "A SqlWeatherSource throws NoDataException for mixed valid and invalid coordinates"() { + given: + def validCoordinate = CosmoWeatherTestData.COORDINATE_193186 + def invalidCoordinate = GeoUtils.buildPoint(999d, 999d) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) + + when: + source.getWeather(timeInterval, [ + validCoordinate, + invalidCoordinate + ]) + + then: + def ex = thrown(NoDataException) + ex.message.contains("No data for given coordinates") + ex.message.contains(invalidCoordinate.toString()) + } } From 3ebda4939304afd6cffe68808a6675484ccd9cb5 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 27 Aug 2025 01:57:30 +0200 Subject: [PATCH 47/53] bug --- .../ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index 55bb3a555..954e1f54b 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -114,7 +114,7 @@ class SqlWeatherSourceCosmoIT extends Specification implements TestContainerHelp then: def ex = thrown(NoDataException) - ex.message.contains("Unable to match any of the provided coordinates to coordinate IDs") + ex.message.contains("No data for given coordinates") ex.message.contains(coordinates.toString()) } From bdd0ae85403d151b6af06d17afb1f6ae0e52df6f Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 27 Aug 2025 02:33:52 +0200 Subject: [PATCH 48/53] Strict fail-fast on partial results --- .../couchbase/CouchbaseWeatherSource.java | 8 ++++++++ .../io/source/csv/CsvWeatherSource.java | 8 ++++++++ .../influxdb/InfluxDbWeatherSource.java | 8 ++++++++ .../io/source/sql/SqlWeatherSource.java | 12 +++++++++++- .../CouchbaseWeatherSourceIconIT.groovy | 19 +++++++++++++++++++ .../csv/CsvWeatherSourceIconTest.groovy | 19 +++++++++++++++++++ .../InfluxDbWeatherSourceIconIT.groovy | 19 +++++++++++++++++++ .../source/sql/SqlWeatherSourceCosmoIT.groovy | 19 +++++++++++++++++++ 8 files changed, 111 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 00761284b..712442d9d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -153,6 +153,14 @@ public Map> getWeather( } } } + + if (coordinateToTimeSeries.size() < coordinates.size()) { + Set missingCoordinates = new HashSet<>(coordinates); + missingCoordinates.removeAll(coordinateToTimeSeries.keySet()); + throw new NoDataException( + "No weather data for coordinates " + missingCoordinates + " in interval " + timeInterval); + } + return coordinateToTimeSeries; } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 45035808d..27fd5a8d6 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -130,6 +130,14 @@ public Map> getWeather( throw new NoDataException( "No weather data found for the given time interval: " + timeInterval); } + + if (result.size() < coordinates.size()) { + Set missingCoordinates = new HashSet<>(coordinates); + missingCoordinates.removeAll(result.keySet()); + throw new NoDataException( + "No weather data for coordinates " + missingCoordinates + " in interval " + timeInterval); + } + return result; } diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 85f0a2d2a..1f9c9b861 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -122,6 +122,14 @@ public Map> getWeather( } } } + + if (coordinateToTimeSeries.size() < coordinates.size()) { + Set missingCoordinates = new HashSet<>(coordinates); + missingCoordinates.removeAll(coordinateToTimeSeries.keySet()); + throw new NoDataException( + "No weather data for coordinates " + missingCoordinates + " in interval " + timeInterval); + } + return coordinateToTimeSeries; } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 30dd703ed..0058d4e3c 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -144,7 +144,17 @@ public Map> getWeather( ps.setTimestamp(3, Timestamp.from(timeInterval.getUpper().toInstant())); })); - return mapWeatherValuesToPoints(timeBasedValues); + Map> result = + mapWeatherValuesToPoints(timeBasedValues); + + if (result.size() < coordinates.size()) { + Set missingCoordinates = new HashSet<>(coordinates); + missingCoordinates.removeAll(result.keySet()); + throw new NoDataException( + "No weather data for coordinates " + missingCoordinates + " in interval " + timeInterval); + } + + return result; } @Override diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy index 8dbdcf8db..e800a4c70 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy @@ -195,6 +195,25 @@ class CouchbaseWeatherSourceIconIT extends Specification implements TestContaine ex.message.contains(invalidCoordinate.toString()) } + def "A CouchbaseWeatherSource throws NoDataException when some valid coordinates have no data in time interval"() { + given: + def validCoordinateWithData = IconWeatherTestData.COORDINATE_67775 + def validCoordinateWithoutData = IconWeatherTestData.COORDINATE_67776 + def futureTimeInterval = new ClosedInterval(IconWeatherTestData.TIME_17H.plusHours(1), IconWeatherTestData.TIME_17H.plusHours(2)) + + when: "requesting weather for valid coordinates but in a future time interval where no data exists" + source.getWeather(futureTimeInterval, [ + validCoordinateWithData, + validCoordinateWithoutData + ]) + + then: "NoDataException is thrown for missing data in interval" + def ex = thrown(NoDataException) + ex.message.contains("No weather data for coordinates") + ex.message.contains("in interval") + ex.message.contains(futureTimeInterval.toString()) + } + def "A CouchbaseWeatherSource throws NoDataException for future date"() { given: def futureDate = IconWeatherTestData.TIME_17H.plusDays(30) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy index 7e3f22f8e..03bfe7d60 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy @@ -340,4 +340,23 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, ex.message.contains("No data for given coordinates") ex.message.contains(invalidCoordinate.toString()) } + + def "A CsvWeatherSource throws NoDataException when some valid coordinates have no data in time interval"() { + given: + def validCoordinateWithData = IconWeatherTestData.COORDINATE_67775 + def validCoordinateWithoutData = IconWeatherTestData.COORDINATE_67776 + def futureTimeInterval = new ClosedInterval(IconWeatherTestData.TIME_17H.plusHours(1), IconWeatherTestData.TIME_17H.plusHours(2)) + + when: "requesting weather for valid coordinates but in a future time interval where no data exists" + source.getWeather(futureTimeInterval, [ + validCoordinateWithData, + validCoordinateWithoutData + ]) + + then: "NoDataException is thrown for missing data in interval" + def ex = thrown(NoDataException) + ex.message.contains("No weather data for coordinates") + ex.message.contains("in interval") + ex.message.contains(futureTimeInterval.toString()) + } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy index 5bee62a39..42834b5af 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy @@ -160,6 +160,25 @@ class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSource ex3.message.contains(invalidCoordinate.toString()) } + def "An InfluxDbWeatherSource throws NoDataException when some valid coordinates have no data in time interval"() { + given: + def validCoordinateWithData = IconWeatherTestData.COORDINATE_67775 + def validCoordinateWithoutData = IconWeatherTestData.COORDINATE_67776 + def futureTimeInterval = new ClosedInterval(IconWeatherTestData.TIME_17H.plusHours(1), IconWeatherTestData.TIME_17H.plusHours(2)) + + when: "requesting weather for valid coordinates but in a future time interval where no data exists" + source.getWeather(futureTimeInterval, [ + validCoordinateWithData, + validCoordinateWithoutData + ]) + + then: "NoDataException is thrown for missing data in interval" + def ex = thrown(NoDataException) + ex.message.contains("No weather data for coordinates") + ex.message.contains("in interval") + ex.message.contains(futureTimeInterval.toString()) + } + def "The InfluxDbWeatherSource returns all time keys after a given time key correctly"() { given: def time = IconWeatherTestData.TIME_15H diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index 954e1f54b..da5991811 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -181,4 +181,23 @@ class SqlWeatherSourceCosmoIT extends Specification implements TestContainerHelp ex.message.contains("No data for given coordinates") ex.message.contains(invalidCoordinate.toString()) } + + def "A SqlWeatherSource throws NoDataException when some valid coordinates have no data in time interval"() { + given: + def validCoordinateWithData = CosmoWeatherTestData.COORDINATE_193186 + def validCoordinateWithoutData = CosmoWeatherTestData.COORDINATE_193187 + def futureTimeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_17H.plusHours(1), CosmoWeatherTestData.TIME_17H.plusHours(2)) + + when: "requesting weather for valid coordinates but in a future time interval where no data exists" + source.getWeather(futureTimeInterval, [ + validCoordinateWithData, + validCoordinateWithoutData + ]) + + then: "NoDataException is thrown for missing data in interval" + def ex = thrown(NoDataException) + ex.message.contains("No weather data for coordinates") + ex.message.contains("in interval") + ex.message.contains(futureTimeInterval.toString()) + } } From 7f6a59cd6b18ebd512128d44cc36edba4dfce0b7 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 27 Aug 2025 02:45:07 +0200 Subject: [PATCH 49/53] Revert "Strict fail-fast on partial results" This reverts commit bdd0ae85403d151b6af06d17afb1f6ae0e52df6f. --- .../couchbase/CouchbaseWeatherSource.java | 8 -------- .../io/source/csv/CsvWeatherSource.java | 8 -------- .../influxdb/InfluxDbWeatherSource.java | 8 -------- .../io/source/sql/SqlWeatherSource.java | 12 +----------- .../CouchbaseWeatherSourceIconIT.groovy | 19 ------------------- .../csv/CsvWeatherSourceIconTest.groovy | 19 ------------------- .../InfluxDbWeatherSourceIconIT.groovy | 19 ------------------- .../source/sql/SqlWeatherSourceCosmoIT.groovy | 19 ------------------- 8 files changed, 1 insertion(+), 111 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 712442d9d..00761284b 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -153,14 +153,6 @@ public Map> getWeather( } } } - - if (coordinateToTimeSeries.size() < coordinates.size()) { - Set missingCoordinates = new HashSet<>(coordinates); - missingCoordinates.removeAll(coordinateToTimeSeries.keySet()); - throw new NoDataException( - "No weather data for coordinates " + missingCoordinates + " in interval " + timeInterval); - } - return coordinateToTimeSeries; } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 27fd5a8d6..45035808d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -130,14 +130,6 @@ public Map> getWeather( throw new NoDataException( "No weather data found for the given time interval: " + timeInterval); } - - if (result.size() < coordinates.size()) { - Set missingCoordinates = new HashSet<>(coordinates); - missingCoordinates.removeAll(result.keySet()); - throw new NoDataException( - "No weather data for coordinates " + missingCoordinates + " in interval " + timeInterval); - } - return result; } diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 1f9c9b861..85f0a2d2a 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -122,14 +122,6 @@ public Map> getWeather( } } } - - if (coordinateToTimeSeries.size() < coordinates.size()) { - Set missingCoordinates = new HashSet<>(coordinates); - missingCoordinates.removeAll(coordinateToTimeSeries.keySet()); - throw new NoDataException( - "No weather data for coordinates " + missingCoordinates + " in interval " + timeInterval); - } - return coordinateToTimeSeries; } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 0058d4e3c..30dd703ed 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -144,17 +144,7 @@ public Map> getWeather( ps.setTimestamp(3, Timestamp.from(timeInterval.getUpper().toInstant())); })); - Map> result = - mapWeatherValuesToPoints(timeBasedValues); - - if (result.size() < coordinates.size()) { - Set missingCoordinates = new HashSet<>(coordinates); - missingCoordinates.removeAll(result.keySet()); - throw new NoDataException( - "No weather data for coordinates " + missingCoordinates + " in interval " + timeInterval); - } - - return result; + return mapWeatherValuesToPoints(timeBasedValues); } @Override diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy index e800a4c70..8dbdcf8db 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy @@ -195,25 +195,6 @@ class CouchbaseWeatherSourceIconIT extends Specification implements TestContaine ex.message.contains(invalidCoordinate.toString()) } - def "A CouchbaseWeatherSource throws NoDataException when some valid coordinates have no data in time interval"() { - given: - def validCoordinateWithData = IconWeatherTestData.COORDINATE_67775 - def validCoordinateWithoutData = IconWeatherTestData.COORDINATE_67776 - def futureTimeInterval = new ClosedInterval(IconWeatherTestData.TIME_17H.plusHours(1), IconWeatherTestData.TIME_17H.plusHours(2)) - - when: "requesting weather for valid coordinates but in a future time interval where no data exists" - source.getWeather(futureTimeInterval, [ - validCoordinateWithData, - validCoordinateWithoutData - ]) - - then: "NoDataException is thrown for missing data in interval" - def ex = thrown(NoDataException) - ex.message.contains("No weather data for coordinates") - ex.message.contains("in interval") - ex.message.contains(futureTimeInterval.toString()) - } - def "A CouchbaseWeatherSource throws NoDataException for future date"() { given: def futureDate = IconWeatherTestData.TIME_17H.plusDays(30) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy index 03bfe7d60..7e3f22f8e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy @@ -340,23 +340,4 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, ex.message.contains("No data for given coordinates") ex.message.contains(invalidCoordinate.toString()) } - - def "A CsvWeatherSource throws NoDataException when some valid coordinates have no data in time interval"() { - given: - def validCoordinateWithData = IconWeatherTestData.COORDINATE_67775 - def validCoordinateWithoutData = IconWeatherTestData.COORDINATE_67776 - def futureTimeInterval = new ClosedInterval(IconWeatherTestData.TIME_17H.plusHours(1), IconWeatherTestData.TIME_17H.plusHours(2)) - - when: "requesting weather for valid coordinates but in a future time interval where no data exists" - source.getWeather(futureTimeInterval, [ - validCoordinateWithData, - validCoordinateWithoutData - ]) - - then: "NoDataException is thrown for missing data in interval" - def ex = thrown(NoDataException) - ex.message.contains("No weather data for coordinates") - ex.message.contains("in interval") - ex.message.contains(futureTimeInterval.toString()) - } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy index 42834b5af..5bee62a39 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy @@ -160,25 +160,6 @@ class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSource ex3.message.contains(invalidCoordinate.toString()) } - def "An InfluxDbWeatherSource throws NoDataException when some valid coordinates have no data in time interval"() { - given: - def validCoordinateWithData = IconWeatherTestData.COORDINATE_67775 - def validCoordinateWithoutData = IconWeatherTestData.COORDINATE_67776 - def futureTimeInterval = new ClosedInterval(IconWeatherTestData.TIME_17H.plusHours(1), IconWeatherTestData.TIME_17H.plusHours(2)) - - when: "requesting weather for valid coordinates but in a future time interval where no data exists" - source.getWeather(futureTimeInterval, [ - validCoordinateWithData, - validCoordinateWithoutData - ]) - - then: "NoDataException is thrown for missing data in interval" - def ex = thrown(NoDataException) - ex.message.contains("No weather data for coordinates") - ex.message.contains("in interval") - ex.message.contains(futureTimeInterval.toString()) - } - def "The InfluxDbWeatherSource returns all time keys after a given time key correctly"() { given: def time = IconWeatherTestData.TIME_15H diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index da5991811..954e1f54b 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -181,23 +181,4 @@ class SqlWeatherSourceCosmoIT extends Specification implements TestContainerHelp ex.message.contains("No data for given coordinates") ex.message.contains(invalidCoordinate.toString()) } - - def "A SqlWeatherSource throws NoDataException when some valid coordinates have no data in time interval"() { - given: - def validCoordinateWithData = CosmoWeatherTestData.COORDINATE_193186 - def validCoordinateWithoutData = CosmoWeatherTestData.COORDINATE_193187 - def futureTimeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_17H.plusHours(1), CosmoWeatherTestData.TIME_17H.plusHours(2)) - - when: "requesting weather for valid coordinates but in a future time interval where no data exists" - source.getWeather(futureTimeInterval, [ - validCoordinateWithData, - validCoordinateWithoutData - ]) - - then: "NoDataException is thrown for missing data in interval" - def ex = thrown(NoDataException) - ex.message.contains("No weather data for coordinates") - ex.message.contains("in interval") - ex.message.contains(futureTimeInterval.toString()) - } } From 183e0c88efce8bf4ca9fac487efc00caa256d003 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 27 Aug 2025 03:10:46 +0200 Subject: [PATCH 50/53] little improvement --- .../java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 30dd703ed..8b962b1f2 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -118,7 +118,7 @@ public Map> getWeather( List invalidCoordinates = coordinates.stream() .filter(coordinate -> idCoordinateSource.getId(coordinate).isEmpty()) - .collect(Collectors.toList()); + .toList(); if (!invalidCoordinates.isEmpty()) { log.warn("Unable to match coordinates {} to coordinate IDs", invalidCoordinates); From bf86b78b8bea1a3400eb0104fea0a0fe1b6cea16 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 27 Aug 2025 03:12:25 +0200 Subject: [PATCH 51/53] removed unnecessary info from exception --- src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java index d7004c8a4..0d24d576e 100644 --- a/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java +++ b/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java @@ -9,8 +9,6 @@ * Exception that should be used whenever no data is received{@link * edu.ie3.datamodel.io.source.DataSource} * - * @version 0.1 - * @since 04.09.24 */ public class NoDataException extends Exception { From 67000bd38b2b74f9612762e598e4e523b4939506 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 27 Aug 2025 03:19:21 +0200 Subject: [PATCH 52/53] spotless... --- src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java index 0d24d576e..cf8aa747e 100644 --- a/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java +++ b/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java @@ -8,7 +8,6 @@ /** * Exception that should be used whenever no data is received{@link * edu.ie3.datamodel.io.source.DataSource} - * */ public class NoDataException extends Exception { From 1be0b8eb690711acdbd1786b34a6b380258baae4 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 27 Aug 2025 15:38:30 +0200 Subject: [PATCH 53/53] removed unnecessary test --- .../couchbase/CouchbaseWeatherSourceCosmoIT.groovy | 12 ------------ .../couchbase/CouchbaseWeatherSourceIconIT.groovy | 12 ------------ 2 files changed, 24 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index 5b35371b2..b8301fec2 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -197,16 +197,4 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements TestContain ex.message.contains("No data for given coordinates") ex.message.contains(invalidCoordinate.toString()) } - - def "A CouchbaseWeatherSource throws NoDataException for future date"() { - given: - def futureDate = CosmoWeatherTestData.TIME_17H.plusDays(30) - - when: - source.getWeather(futureDate, CosmoWeatherTestData.COORDINATE_193186) - - then: - def ex = thrown(NoDataException) - ex.message.contains("Weather document not found") - } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy index 8dbdcf8db..61f125542 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy @@ -194,16 +194,4 @@ class CouchbaseWeatherSourceIconIT extends Specification implements TestContaine ex.message.contains("No data for given coordinates") ex.message.contains(invalidCoordinate.toString()) } - - def "A CouchbaseWeatherSource throws NoDataException for future date"() { - given: - def futureDate = IconWeatherTestData.TIME_17H.plusDays(30) - - when: - source.getWeather(futureDate, IconWeatherTestData.COORDINATE_67775) - - then: - def ex = thrown(NoDataException) - ex.message.contains("Weather document not found") - } }