ready for testing

backend/src/overpass/overpass.py

@@ -1,14 +1,15 @@
 """Module allowing connexion to overpass api and fectch data from OSM."""
-from typing import Literal, List
+import os
 import urllib
+import math
 import logging
 import xml.etree.ElementTree as ET
 
 from .caching_strategy import get_cache_key, CachingStrategy
-from ..constants import OSM_CACHE_DIR
+from ..constants import OSM_CACHE_DIR, OSM_TYPES
 
-logger = logging.getLogger('Overpass')
-osm_types = List[Literal['way', 'node', 'relation']]
+
+RESOLUTION = 0.05
 
 
 class Overpass :
@@ -16,6 +17,9 @@ class Overpass :
     Overpass class to manage the query building and sending to overpass api.
     The caching strategy is a part of this class and initialized upon creation of the Overpass object.
     """
+    logger = logging.getLogger(__name__)
+
+
     def __init__(self, caching_strategy: str = 'XML', cache_dir: str = OSM_CACHE_DIR) :
         """
         Initialize the Overpass instance with the url, headers and caching strategy.
@@ -26,16 +30,14 @@ class Overpass :
 
 
     @classmethod
-    def build_query(self, area: tuple, osm_types: osm_types,
+    def build_query(self, bbox: tuple, osm_types: OSM_TYPES,
                     selector: str, conditions=[], out='center') -> str:
         """
         Constructs a query string for the Overpass API to retrieve OpenStreetMap (OSM) data.
 
         Args:
-            area (tuple): A tuple representing the geographical search area, typically in the format 
-                        (radius, latitude, longitude). The first element is a string like "around:2000" 
-                        specifying the search radius, and the second and third elements represent 
-                        the latitude and longitude as floats or strings.
+            bbox (tuple): A tuple representing the geographical search area, typically in the format 
+                        (lat_min, lon_min, lat_max, lon_max).
             osm_types (list[str]): A list of OSM element types to search for. Must be one or more of 
                                     'Way', 'Node', or 'Relation'.
             selector (str): The key or tag to filter the OSM elements (e.g., 'amenity', 'highway', etc.).
@@ -52,7 +54,6 @@ class Overpass :
         Notes:
             - If no conditions are provided, the query will just use the `selector` to filter the OSM 
             elements without additional constraints.
-            - The search area must always formatted as "(radius, lat, lon)".
         """
         if not isinstance(conditions, list) :
             conditions = [conditions]
@@ -61,15 +62,8 @@ class Overpass :
 
         query = '('
 
-        # Round the radius to nearest 50 and coordinates to generate less queries
-        if area[0] > 500 :
-            search_radius = round(area[0] / 50) * 50
-            loc = tuple((round(area[1], 2), round(area[2], 2)))
-        else :
-            search_radius = round(area[0] / 25) * 25
-            loc = tuple((round(area[1], 3), round(area[2], 3)))
-
-        search_area = f"(around:{search_radius}, {str(loc[0])}, {str(loc[1])})"
+        # convert the bbox to string.
+        bbox_str = f"({','.join(map(str, bbox))})"
 
         if conditions :
             conditions = '(if: ' + ' && '.join(conditions) + ')'
@@ -77,14 +71,15 @@ class Overpass :
             conditions = ''
 
         for elem in osm_types :
-            query += elem + '[' + selector + ']' + conditions + search_area + ';'
+            query += elem + '[' + selector + ']' + conditions + bbox_str + ';'
 
         query += ');' + f'out {out};'
 
         return query
 
 
-    def send_query(self, query: str) -> ET:
+    def send_query(self, bbox: tuple, osm_types: OSM_TYPES,
+                    selector: str, conditions=[], out='center') -> ET:
         """
         Sends the Overpass QL query to the Overpass API and returns the parsed JSON response.
 
@@ -94,18 +89,42 @@ class Overpass :
         Returns:
             dict: The parsed JSON response from the Overpass API, or None if the request fails.
         """
+        # Determine which grid cells overlap with this bounding box.
+        overlapping_cells = self.get_overlapping_cells(bbox)
 
-        # Generate a cache key for the current query
-        cache_key = get_cache_key(query)
+        # Check the cache for any data that overlaps with these cells
+        cell_key_dict = {}
+        for cell in overlapping_cells :
+            for elem in osm_types :
+                key_str = f"{elem}[{selector}]{conditions}({','.join(map(str, cell))})"
+            
+            cell_key_dict[cell] = get_cache_key(key_str)
 
-        # Try to fetch the result from the cache
-        cached_response = self.caching_strategy.get(cache_key)
-        if cached_response is not None :
-            logger.debug("Cache hit.")
-            return cached_response
+        cached_responses = []
+        hollow_cache_keys = []
+
+        # Retrieve the cached data and mark the missing entries as hollow
+        for cell, key in cell_key_dict.items():
+            cached_data = self.caching_strategy.get(key)
+            if cached_data is not None :
+                cached_responses.append(cached_data)
+            else:
+                # Cache miss: Mark the cache key as hollow
+                self.caching_strategy.set_hollow(key, cell, osm_types, selector, conditions, out)
+                hollow_cache_keys.append(key)
+
+        # If there is no missing data, return the cached responses
+        if not hollow_cache_keys :
+            self.logger.debug(f'Cache hit.')
+            return self.combine_cached_data(cached_responses)
+        
+        # TODO If there is SOME missing data : hybrid stuff with partial cache
+        
+        # Build the query string in case of needed overpass query
+        query_str = self.build_query(bbox, osm_types, selector, conditions, out)
 
         # Prepare the data to be sent as POST request, encoded as bytes
-        data = urllib.parse.urlencode({'data': query}).encode('utf-8')
+        data = urllib.parse.urlencode({'data': query_str}).encode('utf-8')
 
         try:
             # Create a Request object with the specified URL, data, and headers
@@ -117,9 +136,7 @@ class Overpass :
                 response_data = response.read().decode('utf-8')
                 root = ET.fromstring(response_data)
 
-                # Cache the response data as an ElementTree root
-                self.caching_strategy.set(cache_key, root)
-                logger.debug("Response data added to cache.")
+                self.logger.debug(f'Cache miss. Fetching data through Overpass\nQuery = {query_str}')
 
                 return root
 
@@ -127,7 +144,108 @@ class Overpass :
             raise ConnectionError(f"Error connecting to Overpass API: {e}") from e
 
 
-def get_base_info(elem: ET.Element, osm_type: osm_types, with_name=False) :
+    def build_query_from_hollow(self, xml_string):
+        """Extract variables from an XML string."""
+        
+        # Parse the XML string into an ElementTree object
+        root = ET.fromstring(xml_string)
+        
+        # Extract values from the XML tree
+        key = root.find('key').text
+        cell = tuple(map(float, root.find('cell').text.strip('()').split(',')))
+        bbox = self.get_bbox_from_grid_cell(cell[0], cell[1])
+        osm_types = root.find('osm_types').text.split(',')
+        selector = root.find('selector').text
+        conditions = root.find('conditions').text.split(',') if root.find('conditions').text != "none" else []
+        out = root.find('out').text
+
+        query_str = self.build_query(bbox, osm_types, selector, conditions, out)
+        
+        return query_str, key
+
+    def get_grid_cell(self, lat: float, lon: float):
+        """
+        Returns the grid cell coordinates for a given latitude and longitude.
+        Each grid cell is 0.05°lat x 0.05°lon resolution in size.
+        """
+        lat_index = math.floor(lat / RESOLUTION)
+        lon_index = math.floor(lon / RESOLUTION)
+        return (lat_index, lon_index)
+
+
+    def get_bbox_from_grid_cell(self, lat_index: int, lon_index: int):
+        """
+        Returns the bounding box for a given grid cell index.
+        Each grid cell is resolution x resolution in size.
+
+        The bounding box is returned as (min_lat, min_lon, max_lat, max_lon).
+        """
+        # Calculate the southwest (min_lat, min_lon) corner of the bounding box
+        min_lat = round(lat_index * RESOLUTION, 2)
+        min_lon = round(lon_index * RESOLUTION, 2)
+
+        # Calculate the northeast (max_lat, max_lon) corner of the bounding box
+        max_lat = round((lat_index + 1) * RESOLUTION, 2)
+        max_lon = round((lon_index + 1) * RESOLUTION, 2)
+
+        return (min_lat, min_lon, max_lat, max_lon)
+
+
+
+    def get_overlapping_cells(self, query_bbox: tuple):
+        """
+        Returns a set of all grid cells that overlap with the given bounding box.
+        """
+        # Extract location from the query bbox
+        lat_min, lon_min, lat_max, lon_max = query_bbox
+
+        min_lat_cell, min_lon_cell = self.get_grid_cell(lat_min, lon_min)
+        max_lat_cell, max_lon_cell = self.get_grid_cell(lat_max, lon_max)
+
+        overlapping_cells = set()
+        for lat_idx in range(min_lat_cell, max_lat_cell + 1):
+            for lon_idx in range(min_lon_cell, max_lon_cell + 1):
+                overlapping_cells.add((lat_idx, lon_idx))
+        
+        return overlapping_cells
+
+
+    def combine_cached_data(self, cached_data_list):
+        """
+        Combines data from multiple cached responses into a single result.
+        """
+        combined_data = ET.Element("osm")
+        for cached_data in cached_data_list:
+            for element in cached_data:
+                combined_data.append(element)
+        return combined_data
+    
+
+    def fill_cache(self, query_str: str, cache_key) :
+        
+        # Prepare the data to be sent as POST request, encoded as bytes
+        data = urllib.parse.urlencode({'data': query_str}).encode('utf-8')
+
+        try:
+            # Create a Request object with the specified URL, data, and headers
+            request = urllib.request.Request(self.overpass_url, data=data, headers=self.headers)
+
+            # Send the request and read the response
+            with urllib.request.urlopen(request) as response:
+                # Read and decode the response
+                response_data = response.read().decode('utf-8')
+                root = ET.fromstring(response_data)
+
+                self.caching_strategy.set(cache_key, root)
+                self.logger.debug(f'Cache set')
+
+        except urllib.error.URLError as e:
+            raise ConnectionError(f"Error connecting to Overpass API: {e}") from e
+
+
+
+
+def get_base_info(elem: ET.Element, osm_type: OSM_TYPES, with_name=False) :
     """
     Extracts base information (coordinates, OSM ID, and optionally a name) from an OSM element.
 
@@ -169,3 +287,26 @@ def get_base_info(elem: ET.Element, osm_type: osm_types, with_name=False) :
         return osm_id, coords, name
     else :
         return osm_id, coords
+
+
+def fill_cache():
+
+    overpass = Overpass(caching_strategy='XML', cache_dir=OSM_CACHE_DIR)
+
+    with os.scandir(OSM_CACHE_DIR) as it:
+        for entry in it:
+            if entry.is_file() and entry.name.startswith('hollow_'):
+                
+                # Read the whole file content as a string
+                with open(entry.path, 'r') as f:
+                    xml_string = f.read()  
+
+                # Build the query and cache key from the hollow XML string
+                query_str, key = overpass.build_query_from_hollow(xml_string)
+                
+                # Fill the cache with the query and key
+                overpass.fill_cache(query_str, key)
+
+                # Now delete the file as the cache is filled
+                os.remove(entry.path)
+